2024-12-11 20:26:28,295 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-11 20:26:28,316 main DEBUG Took 0.012554 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 20:26:28,317 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 20:26:28,317 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 20:26:28,318 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 20:26:28,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,327 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 20:26:28,340 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,342 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,343 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,344 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,345 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,346 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,346 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,347 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,348 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,348 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,348 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,349 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,350 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:26:28,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,351 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 20:26:28,353 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:26:28,354 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 20:26:28,356 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 20:26:28,356 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 20:26:28,358 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 20:26:28,358 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 20:26:28,367 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 20:26:28,370 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 20:26:28,372 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 20:26:28,372 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 20:26:28,372 main DEBUG createAppenders(={Console}) 2024-12-11 20:26:28,373 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-11 20:26:28,373 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-11 20:26:28,373 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-11 20:26:28,374 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 20:26:28,374 main DEBUG OutputStream closed 2024-12-11 20:26:28,375 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 20:26:28,375 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 20:26:28,375 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-11 20:26:28,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 20:26:28,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 20:26:28,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 20:26:28,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 20:26:28,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 20:26:28,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 20:26:28,466 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 20:26:28,466 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 20:26:28,467 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 20:26:28,467 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 20:26:28,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 20:26:28,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 20:26:28,469 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 20:26:28,469 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 20:26:28,470 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 20:26:28,470 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 20:26:28,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 20:26:28,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 20:26:28,474 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 20:26:28,474 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-11 20:26:28,475 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 20:26:28,476 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-11T20:26:28,495 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-11 20:26:28,498 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 20:26:28,499 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T20:26:28,821 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052 2024-12-11T20:26:28,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T20:26:28,858 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881, deleteOnExit=true 2024-12-11T20:26:28,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T20:26:28,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/test.cache.data in system properties and HBase conf 2024-12-11T20:26:28,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T20:26:28,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/hadoop.log.dir in system properties and HBase conf 2024-12-11T20:26:28,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T20:26:28,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T20:26:28,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T20:26:28,965 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T20:26:29,068 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T20:26:29,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T20:26:29,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T20:26:29,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T20:26:29,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T20:26:29,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T20:26:29,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T20:26:29,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T20:26:29,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T20:26:29,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T20:26:29,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/nfs.dump.dir in system properties and HBase conf 2024-12-11T20:26:29,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/java.io.tmpdir in system properties and HBase conf 2024-12-11T20:26:29,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T20:26:29,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T20:26:29,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T20:26:29,920 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T20:26:30,006 INFO [Time-limited test {}] log.Log(170): Logging initialized @2491ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T20:26:30,086 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T20:26:30,151 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T20:26:30,172 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T20:26:30,173 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T20:26:30,174 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T20:26:30,187 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T20:26:30,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/hadoop.log.dir/,AVAILABLE} 2024-12-11T20:26:30,191 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T20:26:30,399 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/java.io.tmpdir/jetty-localhost-37155-hadoop-hdfs-3_4_1-tests_jar-_-any-9388165689860503667/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T20:26:30,407 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:37155} 2024-12-11T20:26:30,407 INFO [Time-limited test {}] server.Server(415): Started @2894ms 2024-12-11T20:26:30,827 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T20:26:30,836 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T20:26:30,838 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T20:26:30,839 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T20:26:30,839 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T20:26:30,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/hadoop.log.dir/,AVAILABLE} 2024-12-11T20:26:30,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T20:26:30,992 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/java.io.tmpdir/jetty-localhost-40299-hadoop-hdfs-3_4_1-tests_jar-_-any-4681359128312704450/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T20:26:30,993 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:40299} 2024-12-11T20:26:30,994 INFO [Time-limited test {}] server.Server(415): Started @3480ms 2024-12-11T20:26:31,061 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T20:26:31,580 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/data/data2/current/BP-555049335-172.17.0.2-1733948789681/current, will proceed with Du for space computation calculation, 2024-12-11T20:26:31,580 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/data/data1/current/BP-555049335-172.17.0.2-1733948789681/current, will proceed with Du for space computation calculation, 2024-12-11T20:26:31,619 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T20:26:31,679 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82d6919a9db2fc2a with lease ID 0x499cd9603972fcd9: Processing first storage report for DS-c2d61507-01b9-4c5c-8a4f-1409e4183a15 from datanode DatanodeRegistration(127.0.0.1:42745, datanodeUuid=a62d8e52-e4a9-4ea7-9c7a-23dd2eff4ff3, infoPort=35505, infoSecurePort=0, ipcPort=42215, storageInfo=lv=-57;cid=testClusterID;nsid=467174717;c=1733948789681) 2024-12-11T20:26:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82d6919a9db2fc2a with lease ID 0x499cd9603972fcd9: from storage DS-c2d61507-01b9-4c5c-8a4f-1409e4183a15 node DatanodeRegistration(127.0.0.1:42745, datanodeUuid=a62d8e52-e4a9-4ea7-9c7a-23dd2eff4ff3, infoPort=35505, infoSecurePort=0, ipcPort=42215, storageInfo=lv=-57;cid=testClusterID;nsid=467174717;c=1733948789681), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T20:26:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82d6919a9db2fc2a with lease ID 0x499cd9603972fcd9: Processing first storage report for DS-7a4c7eb2-9266-45f8-a6ef-4f3d10c581a7 from datanode DatanodeRegistration(127.0.0.1:42745, datanodeUuid=a62d8e52-e4a9-4ea7-9c7a-23dd2eff4ff3, infoPort=35505, infoSecurePort=0, ipcPort=42215, storageInfo=lv=-57;cid=testClusterID;nsid=467174717;c=1733948789681) 2024-12-11T20:26:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82d6919a9db2fc2a with lease ID 0x499cd9603972fcd9: from storage DS-7a4c7eb2-9266-45f8-a6ef-4f3d10c581a7 node DatanodeRegistration(127.0.0.1:42745, datanodeUuid=a62d8e52-e4a9-4ea7-9c7a-23dd2eff4ff3, infoPort=35505, infoSecurePort=0, ipcPort=42215, storageInfo=lv=-57;cid=testClusterID;nsid=467174717;c=1733948789681), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T20:26:31,755 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052 2024-12-11T20:26:31,841 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/zookeeper_0, clientPort=59124, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T20:26:31,854 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59124 2024-12-11T20:26:31,870 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:31,873 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:32,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741825_1001 (size=7) 2024-12-11T20:26:32,612 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 with version=8 2024-12-11T20:26:32,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/hbase-staging 2024-12-11T20:26:32,710 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T20:26:32,965 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=9 2024-12-11T20:26:32,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:26:32,977 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T20:26:32,981 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T20:26:32,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:26:32,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T20:26:33,134 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T20:26:33,200 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T20:26:33,209 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T20:26:33,213 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T20:26:33,240 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 347 (auto-detected) 2024-12-11T20:26:33,241 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-11T20:26:33,261 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43275 2024-12-11T20:26:33,288 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43275 connecting to ZooKeeper ensemble=127.0.0.1:59124 2024-12-11T20:26:33,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432750x0, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T20:26:33,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43275-0x100cb04ee870000 connected 2024-12-11T20:26:33,348 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:33,350 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:33,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:26:33,363 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21, hbase.cluster.distributed=false 2024-12-11T20:26:33,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T20:26:33,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43275 2024-12-11T20:26:33,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43275 2024-12-11T20:26:33,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43275 2024-12-11T20:26:33,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43275 2024-12-11T20:26:33,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43275 2024-12-11T20:26:33,510 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=9 2024-12-11T20:26:33,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:26:33,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T20:26:33,514 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T20:26:33,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:26:33,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T20:26:33,518 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T20:26:33,520 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T20:26:33,521 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39739 2024-12-11T20:26:33,523 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39739 connecting to ZooKeeper ensemble=127.0.0.1:59124 2024-12-11T20:26:33,524 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:33,527 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:33,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397390x0, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T20:26:33,535 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39739-0x100cb04ee870001 connected 2024-12-11T20:26:33,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:26:33,539 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T20:26:33,549 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T20:26:33,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T20:26:33,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T20:26:33,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39739 2024-12-11T20:26:33,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39739 2024-12-11T20:26:33,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39739 2024-12-11T20:26:33,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39739 2024-12-11T20:26:33,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39739 2024-12-11T20:26:33,576 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:43275 2024-12-11T20:26:33,577 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,43275,1733948792764 2024-12-11T20:26:33,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:26:33,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:26:33,586 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,43275,1733948792764 2024-12-11T20:26:33,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:33,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T20:26:33,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:33,610 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T20:26:33,611 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,43275,1733948792764 from backup master directory 2024-12-11T20:26:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,43275,1733948792764 2024-12-11T20:26:33,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:26:33,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:26:33,615 WARN [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T20:26:33,615 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,43275,1733948792764 2024-12-11T20:26:33,618 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T20:26:33,619 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T20:26:33,684 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/hbase.id] with ID: 93cbebb1-e71c-4c57-a5ab-61d89950ad92 2024-12-11T20:26:33,684 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/.tmp/hbase.id 2024-12-11T20:26:33,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741826_1002 (size=42) 2024-12-11T20:26:34,097 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/.tmp/hbase.id]:[hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/hbase.id] 2024-12-11T20:26:34,146 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:26:34,152 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T20:26:34,173 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-11T20:26:34,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:34,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741827_1003 (size=196) 2024-12-11T20:26:34,611 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:26:34,614 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T20:26:34,636 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:26:34,642 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T20:26:34,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741828_1004 (size=1189) 2024-12-11T20:26:35,092 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store 2024-12-11T20:26:35,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741829_1005 (size=34) 2024-12-11T20:26:35,517 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T20:26:35,520 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:26:35,521 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T20:26:35,522 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:26:35,522 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:26:35,523 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T20:26:35,523 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:26:35,524 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:26:35,525 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733948795521Disabling compacts and flushes for region at 1733948795521Disabling writes for close at 1733948795523 (+2 ms)Writing region close event to WAL at 1733948795523Closed at 1733948795523 2024-12-11T20:26:35,527 WARN [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/.initializing 2024-12-11T20:26:35,527 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/WALs/9fe0640122ec,43275,1733948792764 2024-12-11T20:26:35,536 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T20:26:35,550 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C43275%2C1733948792764, suffix=, logDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/WALs/9fe0640122ec,43275,1733948792764, archiveDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/oldWALs, maxLogs=10 2024-12-11T20:26:35,573 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/WALs/9fe0640122ec,43275,1733948792764/9fe0640122ec%2C43275%2C1733948792764.1733948795555, exclude list is [], retry=0 2024-12-11T20:26:35,591 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42745,DS-c2d61507-01b9-4c5c-8a4f-1409e4183a15,DISK] 2024-12-11T20:26:35,594 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T20:26:35,631 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/WALs/9fe0640122ec,43275,1733948792764/9fe0640122ec%2C43275%2C1733948792764.1733948795555 2024-12-11T20:26:35,632 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35505:35505)] 2024-12-11T20:26:35,632 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:26:35,633 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:26:35,636 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,637 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,703 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T20:26:35,707 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:35,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:26:35,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T20:26:35,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:35,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:26:35,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T20:26:35,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:35,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:26:35,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T20:26:35,722 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:35,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:26:35,723 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,726 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,727 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,733 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,734 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,738 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T20:26:35,742 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:26:35,747 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:26:35,748 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68721739, jitterRate=0.02403371036052704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T20:26:35,754 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733948795650Initializing all the Stores at 1733948795653 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948795654 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948795654Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948795655 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948795655Cleaning up temporary data from old regions at 1733948795734 (+79 ms)Region opened successfully at 1733948795754 (+20 ms) 2024-12-11T20:26:35,755 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T20:26:35,788 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268a5ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0 2024-12-11T20:26:35,821 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T20:26:35,833 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T20:26:35,833 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T20:26:35,836 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T20:26:35,837 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T20:26:35,842 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T20:26:35,842 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T20:26:35,868 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T20:26:35,877 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T20:26:35,879 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T20:26:35,882 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T20:26:35,883 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T20:26:35,885 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T20:26:35,887 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T20:26:35,891 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T20:26:35,893 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T20:26:35,894 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T20:26:35,896 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T20:26:35,918 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T20:26:35,919 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T20:26:35,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T20:26:35,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T20:26:35,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:35,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:35,927 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,43275,1733948792764, sessionid=0x100cb04ee870000, setting cluster-up flag (Was=false) 2024-12-11T20:26:35,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:35,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:35,948 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T20:26:35,950 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,43275,1733948792764 2024-12-11T20:26:35,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:35,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:26:35,963 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T20:26:35,965 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,43275,1733948792764 2024-12-11T20:26:35,974 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T20:26:36,054 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T20:26:36,065 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T20:26:36,065 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(746): ClusterId : 93cbebb1-e71c-4c57-a5ab-61d89950ad92 2024-12-11T20:26:36,068 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T20:26:36,072 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T20:26:36,073 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T20:26:36,073 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T20:26:36,077 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T20:26:36,077 DEBUG [RS:0;9fe0640122ec:39739 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23130be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0 2024-12-11T20:26:36,077 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,43275,1733948792764 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T20:26:36,085 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:26:36,085 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:26:36,085 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:26:36,085 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:26:36,085 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10 2024-12-11T20:26:36,086 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,086 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2 2024-12-11T20:26:36,086 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,087 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733948826087 2024-12-11T20:26:36,089 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T20:26:36,090 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T20:26:36,091 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T20:26:36,092 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T20:26:36,093 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:39739 2024-12-11T20:26:36,094 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T20:26:36,094 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T20:26:36,094 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T20:26:36,095 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T20:26:36,096 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T20:26:36,096 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T20:26:36,096 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T20:26:36,095 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,099 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:36,099 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T20:26:36,099 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,43275,1733948792764 with port=39739, startcode=1733948793470 2024-12-11T20:26:36,099 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T20:26:36,100 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T20:26:36,101 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T20:26:36,104 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T20:26:36,104 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T20:26:36,108 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948796106,5,FailOnTimeoutGroup] 2024-12-11T20:26:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741831_1007 (size=1321) 2024-12-11T20:26:36,110 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948796109,5,FailOnTimeoutGroup] 2024-12-11T20:26:36,110 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,110 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T20:26:36,112 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,112 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,112 DEBUG [RS:0;9fe0640122ec:39739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T20:26:36,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36065, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T20:26:36,156 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43275 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,39739,1733948793470 2024-12-11T20:26:36,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43275 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,39739,1733948793470 2024-12-11T20:26:36,173 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:26:36,173 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44245 2024-12-11T20:26:36,174 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T20:26:36,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T20:26:36,179 DEBUG [RS:0;9fe0640122ec:39739 {}] zookeeper.ZKUtil(111): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,39739,1733948793470 2024-12-11T20:26:36,179 WARN [RS:0;9fe0640122ec:39739 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T20:26:36,179 INFO [RS:0;9fe0640122ec:39739 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T20:26:36,179 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470 2024-12-11T20:26:36,181 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,39739,1733948793470] 2024-12-11T20:26:36,208 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T20:26:36,221 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T20:26:36,227 INFO [RS:0;9fe0640122ec:39739 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T20:26:36,227 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,228 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T20:26:36,234 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T20:26:36,235 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,235 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,235 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,236 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,237 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,237 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,237 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:26:36,237 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3 2024-12-11T20:26:36,237 DEBUG [RS:0;9fe0640122ec:39739 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3 2024-12-11T20:26:36,238 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,238 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,239 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,239 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,239 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,239 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,39739,1733948793470-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T20:26:36,259 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T20:26:36,261 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,39739,1733948793470-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,261 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,261 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.Replication(171): 9fe0640122ec,39739,1733948793470 started 2024-12-11T20:26:36,280 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:26:36,281 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,39739,1733948793470, RpcServer on 9fe0640122ec/172.17.0.2:39739, sessionid=0x100cb04ee870001 2024-12-11T20:26:36,281 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T20:26:36,281 DEBUG [RS:0;9fe0640122ec:39739 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,39739,1733948793470 2024-12-11T20:26:36,282 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,39739,1733948793470' 2024-12-11T20:26:36,282 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T20:26:36,283 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T20:26:36,284 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T20:26:36,284 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T20:26:36,284 DEBUG [RS:0;9fe0640122ec:39739 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,39739,1733948793470 2024-12-11T20:26:36,284 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,39739,1733948793470' 2024-12-11T20:26:36,284 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T20:26:36,285 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T20:26:36,286 DEBUG [RS:0;9fe0640122ec:39739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T20:26:36,286 INFO [RS:0;9fe0640122ec:39739 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T20:26:36,286 INFO [RS:0;9fe0640122ec:39739 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T20:26:36,512 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T20:26:36,513 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:26:36,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741832_1008 (size=32) 2024-12-11T20:26:36,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:26:36,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T20:26:36,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T20:26:36,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:36,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:26:36,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T20:26:36,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T20:26:36,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:36,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:26:36,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T20:26:36,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T20:26:36,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:36,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:26:36,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T20:26:36,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T20:26:36,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:26:36,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:26:36,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T20:26:36,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740 2024-12-11T20:26:36,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740 2024-12-11T20:26:36,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T20:26:36,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T20:26:36,550 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T20:26:36,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T20:26:36,557 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:26:36,558 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74676417, jitterRate=0.11276532709598541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T20:26:36,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733948796526Initializing all the Stores at 1733948796528 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948796528Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948796529 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948796529Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948796529Cleaning up temporary data from old regions at 1733948796549 (+20 ms)Region opened successfully at 1733948796561 (+12 ms) 2024-12-11T20:26:36,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T20:26:36,562 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T20:26:36,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T20:26:36,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T20:26:36,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T20:26:36,564 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T20:26:36,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733948796561Disabling compacts and flushes for region at 1733948796561Disabling writes for close at 1733948796562 (+1 ms)Writing region close event to WAL at 1733948796563 (+1 ms)Closed at 1733948796563 2024-12-11T20:26:36,568 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T20:26:36,568 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T20:26:36,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T20:26:36,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T20:26:36,588 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T20:26:36,739 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:37,740 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:38,741 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:39,741 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:40,742 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:41,742 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:42,743 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:43,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-11T20:26:43,198 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-11T20:26:43,744 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:44,744 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:45,745 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:46,746 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:47,746 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:48,747 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:49,747 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:50,748 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:51,749 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:52,749 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:53,750 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:54,751 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:55,751 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:56,752 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:57,753 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:58,753 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:26:59,754 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:00,755 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:01,751 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:27:01,755 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:02,756 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:03,756 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:04,757 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:05,758 WARN [9fe0640122ec:43275 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T20:27:06,292 INFO [RS:0;9fe0640122ec:39739 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T20:27:06,297 INFO [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C39739%2C1733948793470, suffix=, logDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470, archiveDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/oldWALs, maxLogs=32 2024-12-11T20:27:06,317 DEBUG [RS:0;9fe0640122ec:39739 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470/9fe0640122ec%2C39739%2C1733948793470.1733948826300, exclude list is [], retry=0 2024-12-11T20:27:06,321 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42745,DS-c2d61507-01b9-4c5c-8a4f-1409e4183a15,DISK] 2024-12-11T20:27:06,325 INFO [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470/9fe0640122ec%2C39739%2C1733948793470.1733948826300 2024-12-11T20:27:06,325 DEBUG [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35505:35505)] 2024-12-11T20:27:06,511 DEBUG [9fe0640122ec:43275 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-11T20:27:06,523 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:06,530 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,39739,1733948793470, state=OPENING 2024-12-11T20:27:06,535 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T20:27:06,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:27:06,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:27:06,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:27:06,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:27:06,540 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T20:27:06,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:06,719 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T20:27:06,722 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41123, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T20:27:06,735 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T20:27:06,736 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T20:27:06,736 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T20:27:06,740 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C39739%2C1733948793470.meta, suffix=.meta, logDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470, archiveDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/oldWALs, maxLogs=32 2024-12-11T20:27:06,757 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470/9fe0640122ec%2C39739%2C1733948793470.meta.1733948826741.meta, exclude list is [], retry=0 2024-12-11T20:27:06,761 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42745,DS-c2d61507-01b9-4c5c-8a4f-1409e4183a15,DISK] 2024-12-11T20:27:06,765 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/WALs/9fe0640122ec,39739,1733948793470/9fe0640122ec%2C39739%2C1733948793470.meta.1733948826741.meta 2024-12-11T20:27:06,765 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35505:35505)] 2024-12-11T20:27:06,766 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:06,767 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T20:27:06,770 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T20:27:06,775 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T20:27:06,780 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T20:27:06,780 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:06,780 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T20:27:06,780 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T20:27:06,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T20:27:06,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T20:27:06,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:06,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:27:06,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T20:27:06,787 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T20:27:06,788 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:06,788 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:27:06,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T20:27:06,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T20:27:06,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:06,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:27:06,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T20:27:06,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T20:27:06,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:06,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:27:06,793 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T20:27:06,795 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740 2024-12-11T20:27:06,797 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740 2024-12-11T20:27:06,800 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T20:27:06,800 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T20:27:06,801 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T20:27:06,803 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T20:27:06,805 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67237545, jitterRate=0.0019174963235855103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T20:27:06,805 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T20:27:06,806 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733948826781Writing region info on filesystem at 1733948826781Initializing all the Stores at 1733948826783 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948826783Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948826783Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948826783Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948826783Cleaning up temporary data from old regions at 1733948826800 (+17 ms)Running coprocessor post-open hooks at 1733948826805 (+5 ms)Region opened successfully at 1733948826806 (+1 ms) 2024-12-11T20:27:06,814 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733948826709 2024-12-11T20:27:06,827 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T20:27:06,827 INFO [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T20:27:06,829 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:06,831 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,39739,1733948793470, state=OPEN 2024-12-11T20:27:06,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T20:27:06,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T20:27:06,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:27:06,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:27:06,837 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:06,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T20:27:06,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,39739,1733948793470 in 295 msec 2024-12-11T20:27:06,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T20:27:06,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.2690 sec 2024-12-11T20:27:06,852 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T20:27:06,852 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T20:27:06,873 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:27:06,875 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,39739,1733948793470, seqNum=-1] 2024-12-11T20:27:06,898 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:27:06,900 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50727, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:27:06,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9130 sec 2024-12-11T20:27:06,921 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733948826921, completionTime=-1 2024-12-11T20:27:06,924 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-11T20:27:06,924 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T20:27:06,952 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-11T20:27:06,952 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733948886952 2024-12-11T20:27:06,952 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948946952 2024-12-11T20:27:06,953 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-11T20:27:06,955 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,43275,1733948792764-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:06,955 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,43275,1733948792764-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:06,955 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,43275,1733948792764-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:06,957 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:43275, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:06,957 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:06,958 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:06,964 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T20:27:06,985 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.369sec 2024-12-11T20:27:06,986 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T20:27:06,987 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T20:27:06,988 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T20:27:06,989 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T20:27:06,989 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T20:27:06,990 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,43275,1733948792764-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T20:27:07,011 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,43275,1733948792764-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T20:27:07,021 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T20:27:07,021 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T20:27:07,022 INFO [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,43275,1733948792764-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:27:07,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:27:07,026 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T20:27:07,026 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T20:27:07,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,43275,-1 for getting cluster id 2024-12-11T20:27:07,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:27:07,045 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '93cbebb1-e71c-4c57-a5ab-61d89950ad92' 2024-12-11T20:27:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:27:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "93cbebb1-e71c-4c57-a5ab-61d89950ad92" 2024-12-11T20:27:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18214de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:27:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,43275,-1] 2024-12-11T20:27:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:27:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:27:07,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:27:07,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:27:07,060 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:27:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,39739,1733948793470, seqNum=-1] 2024-12-11T20:27:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:27:07,069 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:27:07,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,43275,1733948792764 2024-12-11T20:27:07,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7c601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:27:07,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,43275,-1 for getting cluster id 2024-12-11T20:27:07,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:27:07,094 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '93cbebb1-e71c-4c57-a5ab-61d89950ad92' 2024-12-11T20:27:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:27:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "93cbebb1-e71c-4c57-a5ab-61d89950ad92" 2024-12-11T20:27:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adcb0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:27:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,43275,-1] 2024-12-11T20:27:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:27:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:27:07,097 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:27:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a188c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:27:07,111 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=213, ProcessCount=11, AvailableMemoryMB=828 2024-12-11T20:27:07,117 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:27:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9fe0640122ec,43275,1733948792764 2024-12-11T20:27:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@494ab692 2024-12-11T20:27:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:27:07,128 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:27:07,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-11T20:27:07,148 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:07,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-11T20:27:07,150 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:07,152 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:27:07,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741835_1011 (size=428) 2024-12-11T20:27:07,568 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bdd86b1c3693dd575903a0d36eeb4e01, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:07,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741836_1012 (size=79) 2024-12-11T20:27:07,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:07,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing bdd86b1c3693dd575903a0d36eeb4e01, disabling compactions & flushes 2024-12-11T20:27:07,979 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:07,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:07,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. after waiting 0 ms 2024-12-11T20:27:07,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:07,979 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:07,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for bdd86b1c3693dd575903a0d36eeb4e01: Waiting for close lock at 1733948827979Disabling compacts and flushes for region at 1733948827979Disabling writes for close at 1733948827979Writing region close event to WAL at 1733948827979Closed at 1733948827979 2024-12-11T20:27:07,982 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:07,986 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733948827982"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948827982"}]},"ts":"1733948827982"} 2024-12-11T20:27:07,992 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:07,995 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:07,997 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948827995"}]},"ts":"1733948827995"} 2024-12-11T20:27:08,002 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-11T20:27:08,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bdd86b1c3693dd575903a0d36eeb4e01, ASSIGN}] 2024-12-11T20:27:08,006 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bdd86b1c3693dd575903a0d36eeb4e01, ASSIGN 2024-12-11T20:27:08,008 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bdd86b1c3693dd575903a0d36eeb4e01, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:08,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bdd86b1c3693dd575903a0d36eeb4e01, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:08,165 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bdd86b1c3693dd575903a0d36eeb4e01, ASSIGN because future has completed 2024-12-11T20:27:08,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdd86b1c3693dd575903a0d36eeb4e01, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:27:08,327 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:08,328 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bdd86b1c3693dd575903a0d36eeb4e01, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:08,328 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,328 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:08,328 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,328 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,331 INFO [StoreOpener-bdd86b1c3693dd575903a0d36eeb4e01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,333 INFO [StoreOpener-bdd86b1c3693dd575903a0d36eeb4e01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdd86b1c3693dd575903a0d36eeb4e01 columnFamilyName testFamily 2024-12-11T20:27:08,333 DEBUG [StoreOpener-bdd86b1c3693dd575903a0d36eeb4e01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:08,334 INFO [StoreOpener-bdd86b1c3693dd575903a0d36eeb4e01-1 {}] regionserver.HStore(327): Store=bdd86b1c3693dd575903a0d36eeb4e01/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:08,334 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,335 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,336 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,337 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,337 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,340 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,343 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:08,344 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bdd86b1c3693dd575903a0d36eeb4e01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74265763, jitterRate=0.1066461056470871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:08,344 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:27:08,345 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bdd86b1c3693dd575903a0d36eeb4e01: Running coprocessor pre-open hook at 1733948828329Writing region info on filesystem at 1733948828329Initializing all the Stores at 1733948828330 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948828330Cleaning up temporary data from old regions at 1733948828337 (+7 ms)Running coprocessor post-open hooks at 1733948828344 (+7 ms)Region opened successfully at 1733948828345 (+1 ms) 2024-12-11T20:27:08,347 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01., pid=6, masterSystemTime=1733948828320 2024-12-11T20:27:08,351 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:08,351 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:27:08,352 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bdd86b1c3693dd575903a0d36eeb4e01, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:08,356 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdd86b1c3693dd575903a0d36eeb4e01, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:08,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T20:27:08,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bdd86b1c3693dd575903a0d36eeb4e01, server=9fe0640122ec,39739,1733948793470 in 193 msec 2024-12-11T20:27:08,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T20:27:08,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bdd86b1c3693dd575903a0d36eeb4e01, ASSIGN in 358 msec 2024-12-11T20:27:08,368 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:08,368 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948828368"}]},"ts":"1733948828368"} 2024-12-11T20:27:08,371 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-11T20:27:08,373 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:08,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2350 sec 2024-12-11T20:27:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:27:10,195 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-11T20:27:10,195 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-11T20:27:10,197 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:10,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-11T20:27:10,203 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:10,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-11T20:27:10,206 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:27:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,39739,1733948793470, seqNum=-1] 2024-12-11T20:27:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:27:10,211 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:27:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:10,231 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:10,235 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-11T20:27:10,240 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-11T20:27:10,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-11T20:27:10,247 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-11T20:27:10,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3734180039000850424 2024-12-11T20:27:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948833246, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:10,277 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-11T20:27:10,279 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-11T20:27:10,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:10,281 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:10,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3734180039000850422 2024-12-11T20:27:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948833281, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:10,286 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:10,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3734180039000850421 2024-12-11T20:27:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948833285, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:10,290 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:10,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3734180039000850420 2024-12-11T20:27:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948833289, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:10,294 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:10,306 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2022602898_22 at /127.0.0.1:40970 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=213 (was 213), ProcessCount=11 (was 11), AvailableMemoryMB=809 (was 828) 2024-12-11T20:27:10,319 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=213, ProcessCount=11, AvailableMemoryMB=806 2024-12-11T20:27:10,324 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:10,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-11T20:27:10,328 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:10,328 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:10,328 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-11T20:27:10,330 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:27:10,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741837_1013 (size=414) 2024-12-11T20:27:10,744 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b3ae076303ee4a0b918ae089795dfdea, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:10,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741838_1014 (size=65) 2024-12-11T20:27:11,154 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:11,154 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing b3ae076303ee4a0b918ae089795dfdea, disabling compactions & flushes 2024-12-11T20:27:11,154 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,154 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,154 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. after waiting 0 ms 2024-12-11T20:27:11,154 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,154 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,154 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for b3ae076303ee4a0b918ae089795dfdea: Waiting for close lock at 1733948831154Disabling compacts and flushes for region at 1733948831154Disabling writes for close at 1733948831154Writing region close event to WAL at 1733948831154Closed at 1733948831154 2024-12-11T20:27:11,156 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:11,157 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733948831156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948831156"}]},"ts":"1733948831156"} 2024-12-11T20:27:11,160 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:11,161 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:11,162 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948831161"}]},"ts":"1733948831161"} 2024-12-11T20:27:11,164 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-11T20:27:11,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b3ae076303ee4a0b918ae089795dfdea, ASSIGN}] 2024-12-11T20:27:11,166 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b3ae076303ee4a0b918ae089795dfdea, ASSIGN 2024-12-11T20:27:11,168 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b3ae076303ee4a0b918ae089795dfdea, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:11,319 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b3ae076303ee4a0b918ae089795dfdea, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:11,323 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b3ae076303ee4a0b918ae089795dfdea, ASSIGN because future has completed 2024-12-11T20:27:11,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3ae076303ee4a0b918ae089795dfdea, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:27:11,482 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b3ae076303ee4a0b918ae089795dfdea, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:11,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:11,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,486 INFO [StoreOpener-b3ae076303ee4a0b918ae089795dfdea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,488 INFO [StoreOpener-b3ae076303ee4a0b918ae089795dfdea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3ae076303ee4a0b918ae089795dfdea columnFamilyName testFamily 2024-12-11T20:27:11,488 DEBUG [StoreOpener-b3ae076303ee4a0b918ae089795dfdea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:11,488 INFO [StoreOpener-b3ae076303ee4a0b918ae089795dfdea-1 {}] regionserver.HStore(327): Store=b3ae076303ee4a0b918ae089795dfdea/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:11,489 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,490 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,494 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,497 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:11,498 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened b3ae076303ee4a0b918ae089795dfdea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61968020, jitterRate=-0.07660454511642456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:11,498 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:27:11,500 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b3ae076303ee4a0b918ae089795dfdea: Running coprocessor pre-open hook at 1733948831483Writing region info on filesystem at 1733948831483Initializing all the Stores at 1733948831485 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948831485Cleaning up temporary data from old regions at 1733948831491 (+6 ms)Running coprocessor post-open hooks at 1733948831498 (+7 ms)Region opened successfully at 1733948831500 (+2 ms) 2024-12-11T20:27:11,501 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., pid=9, masterSystemTime=1733948831476 2024-12-11T20:27:11,505 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,505 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:27:11,506 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b3ae076303ee4a0b918ae089795dfdea, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:11,509 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3ae076303ee4a0b918ae089795dfdea, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:11,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-11T20:27:11,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure b3ae076303ee4a0b918ae089795dfdea, server=9fe0640122ec,39739,1733948793470 in 188 msec 2024-12-11T20:27:11,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T20:27:11,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b3ae076303ee4a0b918ae089795dfdea, ASSIGN in 351 msec 2024-12-11T20:27:11,519 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:11,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948831519"}]},"ts":"1733948831519"} 2024-12-11T20:27:11,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-11T20:27:11,523 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:11,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.1990 sec 2024-12-11T20:27:12,313 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:27:12,335 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-11T20:27:12,337 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T20:27:12,337 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-11T20:27:13,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T20:27:13,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:13,198 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T20:27:13,198 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-11T20:27:13,198 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-11T20:27:13,198 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:13,199 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-11T20:27:13,199 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:27:13,365 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-11T20:27:13,365 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-11T20:27:13,366 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:13,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-11T20:27:13,370 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:13,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-11T20:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:13,387 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:13,387 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-11T20:27:13,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T20:27:13,392 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-11T20:27:19,402 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., hostname=9fe0640122ec,39739,1733948793470, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9fe0640122ec:39739 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-11T20:27:19,403 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-11T20:27:19,403 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 from cache 2024-12-11T20:27:19,413 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 213), ProcessCount=11 (was 11), AvailableMemoryMB=790 (was 806) 2024-12-11T20:27:19,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=790 2024-12-11T20:27:19,425 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:19,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-11T20:27:19,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:19,429 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:19,429 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-11T20:27:19,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-11T20:27:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741839_1015 (size=412) 2024-12-11T20:27:19,842 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 65cce501f100c8aafe4ecd17f541aefe, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:19,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741840_1016 (size=63) 2024-12-11T20:27:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-11T20:27:20,252 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:20,252 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 65cce501f100c8aafe4ecd17f541aefe, disabling compactions & flushes 2024-12-11T20:27:20,252 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,252 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,252 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. after waiting 0 ms 2024-12-11T20:27:20,252 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,252 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,252 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 65cce501f100c8aafe4ecd17f541aefe: Waiting for close lock at 1733948840252Disabling compacts and flushes for region at 1733948840252Disabling writes for close at 1733948840252Writing region close event to WAL at 1733948840252Closed at 1733948840252 2024-12-11T20:27:20,254 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:20,255 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733948840254"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948840254"}]},"ts":"1733948840254"} 2024-12-11T20:27:20,258 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:20,259 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:20,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948840259"}]},"ts":"1733948840259"} 2024-12-11T20:27:20,262 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-11T20:27:20,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=65cce501f100c8aafe4ecd17f541aefe, ASSIGN}] 2024-12-11T20:27:20,264 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=65cce501f100c8aafe4ecd17f541aefe, ASSIGN 2024-12-11T20:27:20,265 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=65cce501f100c8aafe4ecd17f541aefe, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:20,416 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=65cce501f100c8aafe4ecd17f541aefe, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:20,420 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=65cce501f100c8aafe4ecd17f541aefe, ASSIGN because future has completed 2024-12-11T20:27:20,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65cce501f100c8aafe4ecd17f541aefe, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-11T20:27:20,580 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 65cce501f100c8aafe4ecd17f541aefe, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:20,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:20,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,583 INFO [StoreOpener-65cce501f100c8aafe4ecd17f541aefe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,585 INFO [StoreOpener-65cce501f100c8aafe4ecd17f541aefe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65cce501f100c8aafe4ecd17f541aefe columnFamilyName testFamily 2024-12-11T20:27:20,585 DEBUG [StoreOpener-65cce501f100c8aafe4ecd17f541aefe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:20,586 INFO [StoreOpener-65cce501f100c8aafe4ecd17f541aefe-1 {}] regionserver.HStore(327): Store=65cce501f100c8aafe4ecd17f541aefe/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:20,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:20,598 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 65cce501f100c8aafe4ecd17f541aefe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66577173, jitterRate=-0.007922813296318054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:20,598 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:27:20,599 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 65cce501f100c8aafe4ecd17f541aefe: Running coprocessor pre-open hook at 1733948840581Writing region info on filesystem at 1733948840581Initializing all the Stores at 1733948840583 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948840583Cleaning up temporary data from old regions at 1733948840589 (+6 ms)Running coprocessor post-open hooks at 1733948840598 (+9 ms)Region opened successfully at 1733948840599 (+1 ms) 2024-12-11T20:27:20,601 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe., pid=12, masterSystemTime=1733948840574 2024-12-11T20:27:20,604 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,604 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:27:20,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=65cce501f100c8aafe4ecd17f541aefe, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:20,610 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65cce501f100c8aafe4ecd17f541aefe, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:20,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-11T20:27:20,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 65cce501f100c8aafe4ecd17f541aefe, server=9fe0640122ec,39739,1733948793470 in 192 msec 2024-12-11T20:27:20,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-11T20:27:20,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=65cce501f100c8aafe4ecd17f541aefe, ASSIGN in 354 msec 2024-12-11T20:27:20,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:20,622 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948840621"}]},"ts":"1733948840621"} 2024-12-11T20:27:20,625 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-11T20:27:20,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:20,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2010 sec 2024-12-11T20:27:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-11T20:27:22,464 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-11T20:27:22,464 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-11T20:27:22,465 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:22,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-11T20:27:22,469 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:22,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-11T20:27:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:22,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:22,484 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-11T20:27:22,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T20:27:22,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T20:27:22,490 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T20:27:22,491 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-11T20:27:23,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-11T20:27:23,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:24,339 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-11T20:27:26,101 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:27:28,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39739 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850411' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T20:27:28,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39739 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41564 deadline: 1733948854499, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850411' does not exist 2024-12-11T20:27:28,503 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9fe0640122ec,39739,1733948793470 for closing scanner id = -3734180039000850411 for 65cce501f100c8aafe4ecd17f541aefe of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850411' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:27:28,506 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=152 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=763 (was 790) 2024-12-11T20:27:28,515 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=152, ProcessCount=11, AvailableMemoryMB=762 2024-12-11T20:27:28,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-11T20:27:28,521 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:28,522 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:28,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-11T20:27:28,523 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:27:28,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741841_1017 (size=423) 2024-12-11T20:27:28,934 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4440fd1fba3c3512abafa8a681ee2568, NAME => 'testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:28,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741842_1018 (size=74) 2024-12-11T20:27:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-11T20:27:29,344 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:29,344 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 4440fd1fba3c3512abafa8a681ee2568, disabling compactions & flushes 2024-12-11T20:27:29,344 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,344 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,344 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. after waiting 0 ms 2024-12-11T20:27:29,344 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,344 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,345 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4440fd1fba3c3512abafa8a681ee2568: Waiting for close lock at 1733948849344Disabling compacts and flushes for region at 1733948849344Disabling writes for close at 1733948849344Writing region close event to WAL at 1733948849344Closed at 1733948849344 2024-12-11T20:27:29,347 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:29,347 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733948849347"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948849347"}]},"ts":"1733948849347"} 2024-12-11T20:27:29,351 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:29,352 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:29,353 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948849352"}]},"ts":"1733948849352"} 2024-12-11T20:27:29,355 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-11T20:27:29,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4440fd1fba3c3512abafa8a681ee2568, ASSIGN}] 2024-12-11T20:27:29,357 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4440fd1fba3c3512abafa8a681ee2568, ASSIGN 2024-12-11T20:27:29,359 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4440fd1fba3c3512abafa8a681ee2568, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:29,510 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4440fd1fba3c3512abafa8a681ee2568, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:29,513 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4440fd1fba3c3512abafa8a681ee2568, ASSIGN because future has completed 2024-12-11T20:27:29,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4440fd1fba3c3512abafa8a681ee2568, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:27:29,673 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,673 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 4440fd1fba3c3512abafa8a681ee2568, NAME => 'testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:29,674 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,674 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:29,674 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,674 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,676 INFO [StoreOpener-4440fd1fba3c3512abafa8a681ee2568-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,679 INFO [StoreOpener-4440fd1fba3c3512abafa8a681ee2568-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4440fd1fba3c3512abafa8a681ee2568 columnFamilyName testFamily 2024-12-11T20:27:29,679 DEBUG [StoreOpener-4440fd1fba3c3512abafa8a681ee2568-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:29,680 INFO [StoreOpener-4440fd1fba3c3512abafa8a681ee2568-1 {}] regionserver.HStore(327): Store=4440fd1fba3c3512abafa8a681ee2568/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:29,680 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,681 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,682 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,682 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,682 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,685 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,688 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:29,689 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 4440fd1fba3c3512abafa8a681ee2568; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62919190, jitterRate=-0.062431007623672485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:29,689 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:27:29,689 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 4440fd1fba3c3512abafa8a681ee2568: Running coprocessor pre-open hook at 1733948849674Writing region info on filesystem at 1733948849674Initializing all the Stores at 1733948849676 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948849676Cleaning up temporary data from old regions at 1733948849683 (+7 ms)Running coprocessor post-open hooks at 1733948849689 (+6 ms)Region opened successfully at 1733948849689 2024-12-11T20:27:29,691 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568., pid=15, masterSystemTime=1733948849667 2024-12-11T20:27:29,694 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,695 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:27:29,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4440fd1fba3c3512abafa8a681ee2568, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:29,699 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4440fd1fba3c3512abafa8a681ee2568, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:29,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-11T20:27:29,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 4440fd1fba3c3512abafa8a681ee2568, server=9fe0640122ec,39739,1733948793470 in 187 msec 2024-12-11T20:27:29,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-11T20:27:29,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4440fd1fba3c3512abafa8a681ee2568, ASSIGN in 349 msec 2024-12-11T20:27:29,708 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:29,709 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948849709"}]},"ts":"1733948849709"} 2024-12-11T20:27:29,712 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-11T20:27:29,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:29,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.1950 sec 2024-12-11T20:27:30,340 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-11T20:27:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:27:31,545 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-11T20:27:31,545 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-11T20:27:31,545 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:31,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-11T20:27:31,550 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:31,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-11T20:27:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:31,563 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:31,563 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-11T20:27:31,566 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-11T20:27:31,568 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-11T20:27:31,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-11T20:27:31,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3734180039000850407 2024-12-11T20:27:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948854568, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:31,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-11T20:27:31,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-11T20:27:31,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:31,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:31,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3734180039000850405 2024-12-11T20:27:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948854574, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:31,578 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:31,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3734180039000850404 2024-12-11T20:27:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948854577, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:31,581 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:31,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3734180039000850403 2024-12-11T20:27:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41564 deadline: 1733948854581, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T20:27:31,585 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T20:27:31,594 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=152 (was 152), ProcessCount=11 (was 11), AvailableMemoryMB=759 (was 762) 2024-12-11T20:27:31,602 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=225, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=152, ProcessCount=11, AvailableMemoryMB=759 2024-12-11T20:27:31,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-11T20:27:31,608 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:31,608 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:31,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-11T20:27:31,609 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-11T20:27:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741843_1019 (size=418) 2024-12-11T20:27:31,752 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:27:32,020 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e3116e57515a6d989750370c39cbc844, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:32,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741844_1020 (size=69) 2024-12-11T20:27:32,429 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:32,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e3116e57515a6d989750370c39cbc844, disabling compactions & flushes 2024-12-11T20:27:32,430 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. after waiting 0 ms 2024-12-11T20:27:32,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,430 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e3116e57515a6d989750370c39cbc844: Waiting for close lock at 1733948852430Disabling compacts and flushes for region at 1733948852430Disabling writes for close at 1733948852430Writing region close event to WAL at 1733948852430Closed at 1733948852430 2024-12-11T20:27:32,432 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:32,432 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733948852432"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948852432"}]},"ts":"1733948852432"} 2024-12-11T20:27:32,435 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:32,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:32,437 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948852436"}]},"ts":"1733948852436"} 2024-12-11T20:27:32,439 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-11T20:27:32,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e3116e57515a6d989750370c39cbc844, ASSIGN}] 2024-12-11T20:27:32,441 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e3116e57515a6d989750370c39cbc844, ASSIGN 2024-12-11T20:27:32,442 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e3116e57515a6d989750370c39cbc844, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:32,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e3116e57515a6d989750370c39cbc844, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:32,597 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e3116e57515a6d989750370c39cbc844, ASSIGN because future has completed 2024-12-11T20:27:32,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3116e57515a6d989750370c39cbc844, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-11T20:27:32,756 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,756 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => e3116e57515a6d989750370c39cbc844, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:32,757 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,757 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:32,757 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,757 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,759 INFO [StoreOpener-e3116e57515a6d989750370c39cbc844-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,761 INFO [StoreOpener-e3116e57515a6d989750370c39cbc844-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3116e57515a6d989750370c39cbc844 columnFamilyName testFamily 2024-12-11T20:27:32,761 DEBUG [StoreOpener-e3116e57515a6d989750370c39cbc844-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:32,762 INFO [StoreOpener-e3116e57515a6d989750370c39cbc844-1 {}] regionserver.HStore(327): Store=e3116e57515a6d989750370c39cbc844/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:32,762 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,763 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,764 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,764 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,765 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,767 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,769 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:32,770 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened e3116e57515a6d989750370c39cbc844; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66160293, jitterRate=-0.014134809374809265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:32,770 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3116e57515a6d989750370c39cbc844 2024-12-11T20:27:32,771 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for e3116e57515a6d989750370c39cbc844: Running coprocessor pre-open hook at 1733948852757Writing region info on filesystem at 1733948852757Initializing all the Stores at 1733948852759 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948852759Cleaning up temporary data from old regions at 1733948852765 (+6 ms)Running coprocessor post-open hooks at 1733948852770 (+5 ms)Region opened successfully at 1733948852771 (+1 ms) 2024-12-11T20:27:32,772 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., pid=18, masterSystemTime=1733948852750 2024-12-11T20:27:32,775 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,775 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:27:32,776 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e3116e57515a6d989750370c39cbc844, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:32,779 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3116e57515a6d989750370c39cbc844, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:32,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-11T20:27:32,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure e3116e57515a6d989750370c39cbc844, server=9fe0640122ec,39739,1733948793470 in 183 msec 2024-12-11T20:27:32,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-11T20:27:32,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e3116e57515a6d989750370c39cbc844, ASSIGN in 343 msec 2024-12-11T20:27:32,786 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:32,786 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948852786"}]},"ts":"1733948852786"} 2024-12-11T20:27:32,789 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-11T20:27:32,790 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:32,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1850 sec 2024-12-11T20:27:33,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-11T20:27:33,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:33,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-11T20:27:33,197 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-11T20:27:34,645 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-11T20:27:34,645 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-11T20:27:34,645 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:34,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-11T20:27:34,650 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:34,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-11T20:27:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:34,664 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:34,665 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-11T20:27:34,668 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T20:27:34,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T20:27:35,176 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:27:35,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9fe0640122ec:39739 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-12-11T20:27:35,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-12-11T20:27:35,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 from cache 2024-12-11T20:27:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-11T20:27:36,341 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-11T20:27:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:36,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T20:27:37,675 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9fe0640122ec:39739 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=988ms, rpcTimeout=981ms 2024-12-11T20:27:37,675 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=988ms, rpcTimeout=981ms 2024-12-11T20:27:37,675 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 from cache 2024-12-11T20:27:37,685 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 225), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 152), ProcessCount=11 (was 11), AvailableMemoryMB=752 (was 759) 2024-12-11T20:27:37,692 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=751 2024-12-11T20:27:37,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-11T20:27:37,699 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:37,699 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:37,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-11T20:27:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:27:37,700 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:37,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741845_1021 (size=416) 2024-12-11T20:27:38,110 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f077f99d334cd858db9d852d667e7a8, NAME => 'testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:38,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741846_1022 (size=67) 2024-12-11T20:27:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-11T20:27:38,520 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:38,520 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0f077f99d334cd858db9d852d667e7a8, disabling compactions & flushes 2024-12-11T20:27:38,520 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,520 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,520 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. after waiting 0 ms 2024-12-11T20:27:38,520 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,520 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,520 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f077f99d334cd858db9d852d667e7a8: Waiting for close lock at 1733948858520Disabling compacts and flushes for region at 1733948858520Disabling writes for close at 1733948858520Writing region close event to WAL at 1733948858520Closed at 1733948858520 2024-12-11T20:27:38,522 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:38,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733948858522"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948858522"}]},"ts":"1733948858522"} 2024-12-11T20:27:38,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:38,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:38,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948858527"}]},"ts":"1733948858527"} 2024-12-11T20:27:38,529 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-11T20:27:38,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0f077f99d334cd858db9d852d667e7a8, ASSIGN}] 2024-12-11T20:27:38,531 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0f077f99d334cd858db9d852d667e7a8, ASSIGN 2024-12-11T20:27:38,532 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0f077f99d334cd858db9d852d667e7a8, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:38,683 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0f077f99d334cd858db9d852d667e7a8, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:38,686 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0f077f99d334cd858db9d852d667e7a8, ASSIGN because future has completed 2024-12-11T20:27:38,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f077f99d334cd858db9d852d667e7a8, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:27:38,845 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,846 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f077f99d334cd858db9d852d667e7a8, NAME => 'testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:38,847 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,847 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:38,847 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,847 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,849 INFO [StoreOpener-0f077f99d334cd858db9d852d667e7a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,851 INFO [StoreOpener-0f077f99d334cd858db9d852d667e7a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f077f99d334cd858db9d852d667e7a8 columnFamilyName testFamily 2024-12-11T20:27:38,851 DEBUG [StoreOpener-0f077f99d334cd858db9d852d667e7a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:38,852 INFO [StoreOpener-0f077f99d334cd858db9d852d667e7a8-1 {}] regionserver.HStore(327): Store=0f077f99d334cd858db9d852d667e7a8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:38,852 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,854 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,854 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,855 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,855 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,857 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,861 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:38,862 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 0f077f99d334cd858db9d852d667e7a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65024071, jitterRate=-0.031065836548805237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:38,862 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:27:38,863 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 0f077f99d334cd858db9d852d667e7a8: Running coprocessor pre-open hook at 1733948858847Writing region info on filesystem at 1733948858847Initializing all the Stores at 1733948858849 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948858849Cleaning up temporary data from old regions at 1733948858855 (+6 ms)Running coprocessor post-open hooks at 1733948858862 (+7 ms)Region opened successfully at 1733948858862 2024-12-11T20:27:38,864 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8., pid=21, masterSystemTime=1733948858840 2024-12-11T20:27:38,867 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,867 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:27:38,868 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0f077f99d334cd858db9d852d667e7a8, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:38,871 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f077f99d334cd858db9d852d667e7a8, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:38,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-11T20:27:38,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f077f99d334cd858db9d852d667e7a8, server=9fe0640122ec,39739,1733948793470 in 185 msec 2024-12-11T20:27:38,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-11T20:27:38,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0f077f99d334cd858db9d852d667e7a8, ASSIGN in 346 msec 2024-12-11T20:27:38,879 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:38,879 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948858879"}]},"ts":"1733948858879"} 2024-12-11T20:27:38,881 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-11T20:27:38,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:38,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1880 sec 2024-12-11T20:27:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:27:40,734 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-11T20:27:40,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-11T20:27:40,734 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:40,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-11T20:27:40,739 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:40,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-11T20:27:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:40,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:40,752 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-11T20:27:40,755 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T20:27:40,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T20:27:40,759 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T20:27:40,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-11T20:27:42,342 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-11T20:27:43,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-11T20:27:43,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:43,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39739 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850392' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T20:27:43,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39739 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41564 deadline: 1733948864768, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850392' does not exist 2024-12-11T20:27:43,769 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9fe0640122ec,39739,1733948793470 for closing scanner id = -3734180039000850392 for 0f077f99d334cd858db9d852d667e7a8 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850392' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:27:43,775 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=118 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=746 (was 751) 2024-12-11T20:27:43,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=118, ProcessCount=11, AvailableMemoryMB=746 2024-12-11T20:27:43,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-11T20:27:43,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:43,789 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:43,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-11T20:27:43,790 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T20:27:43,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741847_1023 (size=411) 2024-12-11T20:27:44,200 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f940ad50107fe2c6e02a78c5100b2427, NAME => 'testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:44,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741848_1024 (size=62) 2024-12-11T20:27:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-11T20:27:44,348 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:27:44,609 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:44,609 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing f940ad50107fe2c6e02a78c5100b2427, disabling compactions & flushes 2024-12-11T20:27:44,609 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,609 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,609 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. after waiting 0 ms 2024-12-11T20:27:44,609 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,609 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,609 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for f940ad50107fe2c6e02a78c5100b2427: Waiting for close lock at 1733948864609Disabling compacts and flushes for region at 1733948864609Disabling writes for close at 1733948864609Writing region close event to WAL at 1733948864609Closed at 1733948864609 2024-12-11T20:27:44,611 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:44,611 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733948864611"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948864611"}]},"ts":"1733948864611"} 2024-12-11T20:27:44,614 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:44,615 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:44,615 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948864615"}]},"ts":"1733948864615"} 2024-12-11T20:27:44,618 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-11T20:27:44,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=f940ad50107fe2c6e02a78c5100b2427, ASSIGN}] 2024-12-11T20:27:44,619 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=f940ad50107fe2c6e02a78c5100b2427, ASSIGN 2024-12-11T20:27:44,620 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=f940ad50107fe2c6e02a78c5100b2427, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:44,771 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=f940ad50107fe2c6e02a78c5100b2427, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:44,774 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=f940ad50107fe2c6e02a78c5100b2427, ASSIGN because future has completed 2024-12-11T20:27:44,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure f940ad50107fe2c6e02a78c5100b2427, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T20:27:44,933 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,934 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => f940ad50107fe2c6e02a78c5100b2427, NAME => 'testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:44,934 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,934 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:44,934 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,934 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,936 INFO [StoreOpener-f940ad50107fe2c6e02a78c5100b2427-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,938 INFO [StoreOpener-f940ad50107fe2c6e02a78c5100b2427-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f940ad50107fe2c6e02a78c5100b2427 columnFamilyName testFamily 2024-12-11T20:27:44,938 DEBUG [StoreOpener-f940ad50107fe2c6e02a78c5100b2427-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:44,939 INFO [StoreOpener-f940ad50107fe2c6e02a78c5100b2427-1 {}] regionserver.HStore(327): Store=f940ad50107fe2c6e02a78c5100b2427/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:44,939 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,940 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,940 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,941 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,941 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,943 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,946 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:44,947 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened f940ad50107fe2c6e02a78c5100b2427; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72745416, jitterRate=0.0839911699295044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:44,947 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:27:44,947 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for f940ad50107fe2c6e02a78c5100b2427: Running coprocessor pre-open hook at 1733948864934Writing region info on filesystem at 1733948864934Initializing all the Stores at 1733948864936 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948864936Cleaning up temporary data from old regions at 1733948864941 (+5 ms)Running coprocessor post-open hooks at 1733948864947 (+6 ms)Region opened successfully at 1733948864947 2024-12-11T20:27:44,949 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427., pid=24, masterSystemTime=1733948864928 2024-12-11T20:27:44,952 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,952 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:27:44,952 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=f940ad50107fe2c6e02a78c5100b2427, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:44,955 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure f940ad50107fe2c6e02a78c5100b2427, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:44,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-11T20:27:44,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure f940ad50107fe2c6e02a78c5100b2427, server=9fe0640122ec,39739,1733948793470 in 182 msec 2024-12-11T20:27:44,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-11T20:27:44,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=f940ad50107fe2c6e02a78c5100b2427, ASSIGN in 341 msec 2024-12-11T20:27:44,963 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:44,963 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948864963"}]},"ts":"1733948864963"} 2024-12-11T20:27:44,966 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-11T20:27:44,967 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:44,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1810 sec 2024-12-11T20:27:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T20:27:46,815 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-11T20:27:46,815 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-11T20:27:46,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:46,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-11T20:27:46,820 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:46,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-11T20:27:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:46,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:46,833 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-11T20:27:46,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T20:27:46,837 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T20:27:46,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T20:27:46,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-11T20:27:47,914 INFO [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-11T20:27:47,914 INFO [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-11T20:27:48,343 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-11T20:27:49,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39739 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850388' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T20:27:49,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39739 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41564 deadline: 1733948870848, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850388' does not exist 2024-12-11T20:27:49,850 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9fe0640122ec,39739,1733948793470 for closing scanner id = -3734180039000850388 for f940ad50107fe2c6e02a78c5100b2427 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850388' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:27:49,855 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=109 (was 118), ProcessCount=11 (was 11), AvailableMemoryMB=741 (was 746) 2024-12-11T20:27:49,863 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=109, ProcessCount=11, AvailableMemoryMB=741 2024-12-11T20:27:49,865 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:49,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-11T20:27:49,869 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:49,869 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:49,869 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-11T20:27:49,870 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:27:49,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741849_1025 (size=419) 2024-12-11T20:27:50,280 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9680151e407195902f12fd73ae967fab, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:50,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741850_1026 (size=70) 2024-12-11T20:27:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-11T20:27:50,436 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:27:50,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:50,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9680151e407195902f12fd73ae967fab, disabling compactions & flushes 2024-12-11T20:27:50,689 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:50,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:50,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. after waiting 0 ms 2024-12-11T20:27:50,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:50,689 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:50,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9680151e407195902f12fd73ae967fab: Waiting for close lock at 1733948870689Disabling compacts and flushes for region at 1733948870689Disabling writes for close at 1733948870689Writing region close event to WAL at 1733948870689Closed at 1733948870689 2024-12-11T20:27:50,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:50,691 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733948870691"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948870691"}]},"ts":"1733948870691"} 2024-12-11T20:27:50,694 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:50,695 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:50,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948870695"}]},"ts":"1733948870695"} 2024-12-11T20:27:50,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-11T20:27:50,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9680151e407195902f12fd73ae967fab, ASSIGN}] 2024-12-11T20:27:50,700 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9680151e407195902f12fd73ae967fab, ASSIGN 2024-12-11T20:27:50,701 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9680151e407195902f12fd73ae967fab, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:50,852 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9680151e407195902f12fd73ae967fab, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:50,855 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9680151e407195902f12fd73ae967fab, ASSIGN because future has completed 2024-12-11T20:27:50,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9680151e407195902f12fd73ae967fab, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:27:51,014 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:51,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 9680151e407195902f12fd73ae967fab, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:27:51,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:51,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,017 INFO [StoreOpener-9680151e407195902f12fd73ae967fab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,019 INFO [StoreOpener-9680151e407195902f12fd73ae967fab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9680151e407195902f12fd73ae967fab columnFamilyName testFamily 2024-12-11T20:27:51,019 DEBUG [StoreOpener-9680151e407195902f12fd73ae967fab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:51,020 INFO [StoreOpener-9680151e407195902f12fd73ae967fab-1 {}] regionserver.HStore(327): Store=9680151e407195902f12fd73ae967fab/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:27:51,020 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,025 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,027 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:27:51,028 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 9680151e407195902f12fd73ae967fab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60030981, jitterRate=-0.10546867549419403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:27:51,028 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9680151e407195902f12fd73ae967fab 2024-12-11T20:27:51,029 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 9680151e407195902f12fd73ae967fab: Running coprocessor pre-open hook at 1733948871015Writing region info on filesystem at 1733948871015Initializing all the Stores at 1733948871017 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948871017Cleaning up temporary data from old regions at 1733948871022 (+5 ms)Running coprocessor post-open hooks at 1733948871028 (+6 ms)Region opened successfully at 1733948871029 (+1 ms) 2024-12-11T20:27:51,030 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., pid=27, masterSystemTime=1733948871008 2024-12-11T20:27:51,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:51,033 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:27:51,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9680151e407195902f12fd73ae967fab, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:51,036 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9680151e407195902f12fd73ae967fab, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:27:51,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-11T20:27:51,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 9680151e407195902f12fd73ae967fab, server=9fe0640122ec,39739,1733948793470 in 182 msec 2024-12-11T20:27:51,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-11T20:27:51,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9680151e407195902f12fd73ae967fab, ASSIGN in 341 msec 2024-12-11T20:27:51,042 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:27:51,043 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948871042"}]},"ts":"1733948871042"} 2024-12-11T20:27:51,045 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-11T20:27:51,046 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:27:51,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1810 sec 2024-12-11T20:27:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:27:52,905 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-11T20:27:52,905 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-11T20:27:52,905 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:52,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-11T20:27:52,909 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:27:52,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-11T20:27:52,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:27:52,922 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:27:52,922 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-11T20:27:52,924 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T20:27:52,926 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-11T20:27:53,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-11T20:27:53,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:53,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-11T20:27:53,197 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T20:27:54,345 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-11T20:27:56,516 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:27:58,925 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., hostname=9fe0640122ec,39739,1733948793470, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9fe0640122ec:39739 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-12-11T20:27:58,925 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-12-11T20:27:58,926 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 from cache 2024-12-11T20:27:58,935 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=92 (was 109), ProcessCount=11 (was 11), AvailableMemoryMB=734 (was 741) 2024-12-11T20:27:58,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=92, ProcessCount=11, AvailableMemoryMB=734 2024-12-11T20:27:58,946 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:27:58,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-11T20:27:58,949 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:27:58,949 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:27:58,949 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-11T20:27:58,950 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:27:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T20:27:58,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741851_1027 (size=407) 2024-12-11T20:27:59,361 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fdc8ac339ad1b89b0b29e3734ea77e36, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:27:59,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741852_1028 (size=58) 2024-12-11T20:27:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-11T20:27:59,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:27:59,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing fdc8ac339ad1b89b0b29e3734ea77e36, disabling compactions & flushes 2024-12-11T20:27:59,770 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:27:59,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:27:59,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. after waiting 0 ms 2024-12-11T20:27:59,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:27:59,770 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:27:59,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for fdc8ac339ad1b89b0b29e3734ea77e36: Waiting for close lock at 1733948879770Disabling compacts and flushes for region at 1733948879770Disabling writes for close at 1733948879770Writing region close event to WAL at 1733948879770Closed at 1733948879770 2024-12-11T20:27:59,772 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:27:59,772 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733948879772"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948879772"}]},"ts":"1733948879772"} 2024-12-11T20:27:59,775 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:27:59,776 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:27:59,777 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948879776"}]},"ts":"1733948879776"} 2024-12-11T20:27:59,780 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-11T20:27:59,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fdc8ac339ad1b89b0b29e3734ea77e36, ASSIGN}] 2024-12-11T20:27:59,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fdc8ac339ad1b89b0b29e3734ea77e36, ASSIGN 2024-12-11T20:27:59,782 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fdc8ac339ad1b89b0b29e3734ea77e36, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:27:59,933 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fdc8ac339ad1b89b0b29e3734ea77e36, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:27:59,936 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fdc8ac339ad1b89b0b29e3734ea77e36, ASSIGN because future has completed 2024-12-11T20:27:59,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdc8ac339ad1b89b0b29e3734ea77e36, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:27:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T20:28:00,095 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:00,095 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => fdc8ac339ad1b89b0b29e3734ea77e36, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:28:00,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:28:00,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,098 INFO [StoreOpener-fdc8ac339ad1b89b0b29e3734ea77e36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,100 INFO [StoreOpener-fdc8ac339ad1b89b0b29e3734ea77e36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdc8ac339ad1b89b0b29e3734ea77e36 columnFamilyName testFamily 2024-12-11T20:28:00,100 DEBUG [StoreOpener-fdc8ac339ad1b89b0b29e3734ea77e36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:28:00,100 INFO [StoreOpener-fdc8ac339ad1b89b0b29e3734ea77e36-1 {}] regionserver.HStore(327): Store=fdc8ac339ad1b89b0b29e3734ea77e36/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:28:00,101 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,101 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,104 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,107 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:28:00,107 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened fdc8ac339ad1b89b0b29e3734ea77e36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58796243, jitterRate=-0.1238677054643631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:28:00,107 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:00,108 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for fdc8ac339ad1b89b0b29e3734ea77e36: Running coprocessor pre-open hook at 1733948880096Writing region info on filesystem at 1733948880096Initializing all the Stores at 1733948880097 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948880097Cleaning up temporary data from old regions at 1733948880102 (+5 ms)Running coprocessor post-open hooks at 1733948880108 (+6 ms)Region opened successfully at 1733948880108 2024-12-11T20:28:00,109 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36., pid=30, masterSystemTime=1733948880089 2024-12-11T20:28:00,112 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:00,112 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:00,113 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fdc8ac339ad1b89b0b29e3734ea77e36, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:28:00,115 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdc8ac339ad1b89b0b29e3734ea77e36, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:28:00,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-11T20:28:00,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure fdc8ac339ad1b89b0b29e3734ea77e36, server=9fe0640122ec,39739,1733948793470 in 179 msec 2024-12-11T20:28:00,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-11T20:28:00,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fdc8ac339ad1b89b0b29e3734ea77e36, ASSIGN in 339 msec 2024-12-11T20:28:00,123 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:28:00,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948880123"}]},"ts":"1733948880123"} 2024-12-11T20:28:00,125 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-11T20:28:00,126 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:28:00,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1800 sec 2024-12-11T20:28:00,347 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-11T20:28:01,752 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:28:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T20:28:01,995 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-11T20:28:01,995 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-11T20:28:01,995 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:28:01,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-11T20:28:01,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:28:01,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-11T20:28:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:28:02,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:28:02,013 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-11T20:28:02,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T20:28:02,018 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T20:28:02,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T20:28:02,019 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-11T20:28:03,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-11T20:28:03,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-11T20:28:05,597 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:28:07,052 DEBUG [master/9fe0640122ec:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-11T20:28:08,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39739 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850380' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T20:28:08,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39739 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41564 deadline: 1733948894026, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850380' does not exist 2024-12-11T20:28:08,030 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9fe0640122ec,39739,1733948793470 for closing scanner id = -3734180039000850380 for fdc8ac339ad1b89b0b29e3734ea77e36 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3734180039000850380' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:28:08,036 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=102 (was 92) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2048 (was 734) - AvailableMemoryMB LEAK? - 2024-12-11T20:28:08,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=102, ProcessCount=11, AvailableMemoryMB=2048 2024-12-11T20:28:08,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:28:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-11T20:28:08,052 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:28:08,052 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:28:08,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-11T20:28:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T20:28:08,054 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:28:08,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741853_1029 (size=423) 2024-12-11T20:28:08,465 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6e1f0c5a512aa504f84ad541d42d8c2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21 2024-12-11T20:28:08,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741854_1030 (size=74) 2024-12-11T20:28:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-11T20:28:08,875 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:28:08,876 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing e6e1f0c5a512aa504f84ad541d42d8c2, disabling compactions & flushes 2024-12-11T20:28:08,876 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:08,876 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:08,876 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. after waiting 0 ms 2024-12-11T20:28:08,876 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:08,876 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:08,876 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6e1f0c5a512aa504f84ad541d42d8c2: Waiting for close lock at 1733948888876Disabling compacts and flushes for region at 1733948888876Disabling writes for close at 1733948888876Writing region close event to WAL at 1733948888876Closed at 1733948888876 2024-12-11T20:28:08,878 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:28:08,879 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733948888878"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948888878"}]},"ts":"1733948888878"} 2024-12-11T20:28:08,882 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:28:08,883 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:28:08,884 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948888883"}]},"ts":"1733948888883"} 2024-12-11T20:28:08,886 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-11T20:28:08,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=e6e1f0c5a512aa504f84ad541d42d8c2, ASSIGN}] 2024-12-11T20:28:08,888 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=e6e1f0c5a512aa504f84ad541d42d8c2, ASSIGN 2024-12-11T20:28:08,889 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=e6e1f0c5a512aa504f84ad541d42d8c2, ASSIGN; state=OFFLINE, location=9fe0640122ec,39739,1733948793470; forceNewPlan=false, retain=false 2024-12-11T20:28:09,040 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=e6e1f0c5a512aa504f84ad541d42d8c2, regionState=OPENING, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:28:09,044 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=e6e1f0c5a512aa504f84ad541d42d8c2, ASSIGN because future has completed 2024-12-11T20:28:09,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6e1f0c5a512aa504f84ad541d42d8c2, server=9fe0640122ec,39739,1733948793470}] 2024-12-11T20:28:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T20:28:09,204 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:09,204 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => e6e1f0c5a512aa504f84ad541d42d8c2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:28:09,205 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,205 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:28:09,205 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,205 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,207 INFO [StoreOpener-e6e1f0c5a512aa504f84ad541d42d8c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,209 INFO [StoreOpener-e6e1f0c5a512aa504f84ad541d42d8c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6e1f0c5a512aa504f84ad541d42d8c2 columnFamilyName testFamily 2024-12-11T20:28:09,210 DEBUG [StoreOpener-e6e1f0c5a512aa504f84ad541d42d8c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:28:09,210 INFO [StoreOpener-e6e1f0c5a512aa504f84ad541d42d8c2-1 {}] regionserver.HStore(327): Store=e6e1f0c5a512aa504f84ad541d42d8c2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:28:09,211 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,212 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,212 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,213 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,213 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,215 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,218 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:28:09,218 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened e6e1f0c5a512aa504f84ad541d42d8c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61549380, jitterRate=-0.08284276723861694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T20:28:09,219 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:09,219 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for e6e1f0c5a512aa504f84ad541d42d8c2: Running coprocessor pre-open hook at 1733948889205Writing region info on filesystem at 1733948889205Initializing all the Stores at 1733948889207 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948889207Cleaning up temporary data from old regions at 1733948889213 (+6 ms)Running coprocessor post-open hooks at 1733948889219 (+6 ms)Region opened successfully at 1733948889219 2024-12-11T20:28:09,220 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., pid=33, masterSystemTime=1733948889197 2024-12-11T20:28:09,223 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:09,223 INFO [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:09,224 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=e6e1f0c5a512aa504f84ad541d42d8c2, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,39739,1733948793470 2024-12-11T20:28:09,226 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6e1f0c5a512aa504f84ad541d42d8c2, server=9fe0640122ec,39739,1733948793470 because future has completed 2024-12-11T20:28:09,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-11T20:28:09,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure e6e1f0c5a512aa504f84ad541d42d8c2, server=9fe0640122ec,39739,1733948793470 in 184 msec 2024-12-11T20:28:09,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-11T20:28:09,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=e6e1f0c5a512aa504f84ad541d42d8c2, ASSIGN in 343 msec 2024-12-11T20:28:09,234 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:28:09,234 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948889234"}]},"ts":"1733948889234"} 2024-12-11T20:28:09,236 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-11T20:28:09,237 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:28:09,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1880 sec 2024-12-11T20:28:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43275 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T20:28:11,085 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-11T20:28:11,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-11T20:28:11,085 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:28:11,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-11T20:28:11,092 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T20:28:11,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-11T20:28:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:28:11,108 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T20:28:11,108 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-11T20:28:11,112 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T20:28:11,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T20:28:12,115 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9fe0640122ec:39739 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-12-11T20:28:12,115 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-12-11T20:28:12,115 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 from cache 2024-12-11T20:28:12,349 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-11T20:28:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-11T20:28:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2] 2024-12-11T20:28:13,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39739 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T20:28:13,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-11T20:28:13,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T20:28:14,115 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9fe0640122ec:39739 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=978ms, rpcTimeout=975ms 2024-12-11T20:28:14,115 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=978ms, rpcTimeout=975ms 2024-12-11T20:28:14,116 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., hostname=9fe0640122ec,39739,1733948793470, seqNum=2 from cache 2024-12-11T20:28:14,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=102 (was 102), ProcessCount=11 (was 11), AvailableMemoryMB=1447 (was 2048) 2024-12-11T20:28:14,127 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:28:14,127 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:28:14,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:14,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:14,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:28:14,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T20:28:14,133 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:28:14,133 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:28:14,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:14,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:14,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:28:14,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T20:28:14,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2048927636, stopped=false 2024-12-11T20:28:14,134 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,43275,1733948792764 2024-12-11T20:28:14,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T20:28:14,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T20:28:14,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:28:14,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:28:14,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T20:28:14,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:28:14,137 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:28:14,138 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:28:14,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:28:14,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:14,138 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,39739,1733948793470' ***** 2024-12-11T20:28:14,138 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T20:28:14,138 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T20:28:14,139 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T20:28:14,139 INFO [RS:0;9fe0640122ec:39739 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T20:28:14,139 INFO [RS:0;9fe0640122ec:39739 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T20:28:14,139 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for b3ae076303ee4a0b918ae089795dfdea 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for f940ad50107fe2c6e02a78c5100b2427 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for bdd86b1c3693dd575903a0d36eeb4e01 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for e3116e57515a6d989750370c39cbc844 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for 0f077f99d334cd858db9d852d667e7a8 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for e6e1f0c5a512aa504f84ad541d42d8c2 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for 65cce501f100c8aafe4ecd17f541aefe 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for 4440fd1fba3c3512abafa8a681ee2568 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for 9680151e407195902f12fd73ae967fab 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,39739,1733948793470 2024-12-11T20:28:14,140 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b3ae076303ee4a0b918ae089795dfdea, disabling compactions & flushes 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T20:28:14,140 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:28:14,140 INFO [RS:0;9fe0640122ec:39739 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:39739. 2024-12-11T20:28:14,140 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:28:14,140 DEBUG [RS:0;9fe0640122ec:39739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:28:14,140 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. after waiting 0 ms 2024-12-11T20:28:14,140 DEBUG [RS:0;9fe0640122ec:39739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:14,140 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:28:14,141 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T20:28:14,141 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T20:28:14,141 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T20:28:14,141 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T20:28:14,141 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-11T20:28:14,141 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T20:28:14,141 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T20:28:14,141 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 1588230740, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, b3ae076303ee4a0b918ae089795dfdea, bdd86b1c3693dd575903a0d36eeb4e01, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, f940ad50107fe2c6e02a78c5100b2427, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:14,141 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T20:28:14,141 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T20:28:14,142 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T20:28:14,144 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b3ae076303ee4a0b918ae089795dfdea 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:14,144 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-11T20:28:14,215 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/.tmp/testFamily/80a34be7112f44188d16d79b81a991a2 is 61, key is row-0/testFamily:testQualifier/1733948833376/Put/seqid=0 2024-12-11T20:28:14,221 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/info/3ed440b3b72e45998734e2b7f4b86a81 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01./info:regioninfo/1733948828352/Put/seqid=0 2024-12-11T20:28:14,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741856_1032 (size=20564) 2024-12-11T20:28:14,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741855_1031 (size=5049) 2024-12-11T20:28:14,244 INFO [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T20:28:14,244 INFO [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T20:28:14,342 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 1588230740, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, b3ae076303ee4a0b918ae089795dfdea, bdd86b1c3693dd575903a0d36eeb4e01, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, f940ad50107fe2c6e02a78c5100b2427, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:14,542 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 1588230740, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, b3ae076303ee4a0b918ae089795dfdea, bdd86b1c3693dd575903a0d36eeb4e01, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, f940ad50107fe2c6e02a78c5100b2427, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:14,636 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/info/3ed440b3b72e45998734e2b7f4b86a81 2024-12-11T20:28:14,639 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/.tmp/testFamily/80a34be7112f44188d16d79b81a991a2 2024-12-11T20:28:14,639 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-11T20:28:14,684 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/.tmp/testFamily/80a34be7112f44188d16d79b81a991a2 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/testFamily/80a34be7112f44188d16d79b81a991a2 2024-12-11T20:28:14,695 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/testFamily/80a34be7112f44188d16d79b81a991a2, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:14,704 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/ns/8b27e3cb56554de9a57bc7a487e4b84f is 43, key is default/ns:d/1733948826905/Put/seqid=0 2024-12-11T20:28:14,706 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:28:14,708 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b3ae076303ee4a0b918ae089795dfdea in 560ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:14,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741857_1033 (size=5153) 2024-12-11T20:28:14,728 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScanner/b3ae076303ee4a0b918ae089795dfdea/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:14,733 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:28:14,734 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b3ae076303ee4a0b918ae089795dfdea: Waiting for close lock at 1733948894140Running coprocessor pre-close hooks at 1733948894140Disabling compacts and flushes for region at 1733948894140Disabling writes for close at 1733948894140Obtaining lock to block concurrent updates at 1733948894144 (+4 ms)Preparing flush snapshotting stores in b3ae076303ee4a0b918ae089795dfdea at 1733948894144Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948894152 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. at 1733948894154 (+2 ms)Flushing b3ae076303ee4a0b918ae089795dfdea/testFamily: creating writer at 1733948894156 (+2 ms)Flushing b3ae076303ee4a0b918ae089795dfdea/testFamily: appending metadata at 1733948894207 (+51 ms)Flushing b3ae076303ee4a0b918ae089795dfdea/testFamily: closing flushed file at 1733948894211 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@639bfc24: reopening flushed file at 1733948894681 (+470 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b3ae076303ee4a0b918ae089795dfdea in 560ms, sequenceid=8, compaction requested=false at 1733948894708 (+27 ms)Writing region close event to WAL at 1733948894722 (+14 ms)Running coprocessor post-close hooks at 1733948894731 (+9 ms)Closed at 1733948894733 (+2 ms) 2024-12-11T20:28:14,734 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733948830323.b3ae076303ee4a0b918ae089795dfdea. 2024-12-11T20:28:14,734 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f940ad50107fe2c6e02a78c5100b2427, disabling compactions & flushes 2024-12-11T20:28:14,735 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:28:14,735 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:28:14,735 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. after waiting 0 ms 2024-12-11T20:28:14,735 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:28:14,735 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f940ad50107fe2c6e02a78c5100b2427 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:14,742 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 1588230740, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, bdd86b1c3693dd575903a0d36eeb4e01, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, f940ad50107fe2c6e02a78c5100b2427, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:14,758 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/.tmp/testFamily/f06bf0267574454ab9117da2463d4d48 is 61, key is row-0/testFamily:testQualifier/1733948866824/Put/seqid=0 2024-12-11T20:28:14,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741858_1034 (size=5049) 2024-12-11T20:28:14,764 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/.tmp/testFamily/f06bf0267574454ab9117da2463d4d48 2024-12-11T20:28:14,775 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/.tmp/testFamily/f06bf0267574454ab9117da2463d4d48 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/testFamily/f06bf0267574454ab9117da2463d4d48 2024-12-11T20:28:14,784 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/testFamily/f06bf0267574454ab9117da2463d4d48, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:14,785 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f940ad50107fe2c6e02a78c5100b2427 in 50ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:14,790 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNext/f940ad50107fe2c6e02a78c5100b2427/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:14,791 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:28:14,791 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f940ad50107fe2c6e02a78c5100b2427: Waiting for close lock at 1733948894734Running coprocessor pre-close hooks at 1733948894734Disabling compacts and flushes for region at 1733948894734Disabling writes for close at 1733948894735 (+1 ms)Obtaining lock to block concurrent updates at 1733948894735Preparing flush snapshotting stores in f940ad50107fe2c6e02a78c5100b2427 at 1733948894735Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948894735Flushing stores of testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. at 1733948894736 (+1 ms)Flushing f940ad50107fe2c6e02a78c5100b2427/testFamily: creating writer at 1733948894736Flushing f940ad50107fe2c6e02a78c5100b2427/testFamily: appending metadata at 1733948894757 (+21 ms)Flushing f940ad50107fe2c6e02a78c5100b2427/testFamily: closing flushed file at 1733948894757Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a23619a: reopening flushed file at 1733948894773 (+16 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f940ad50107fe2c6e02a78c5100b2427 in 50ms, sequenceid=8, compaction requested=false at 1733948894785 (+12 ms)Writing region close event to WAL at 1733948894785Running coprocessor post-close hooks at 1733948894791 (+6 ms)Closed at 1733948894791 2024-12-11T20:28:14,791 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733948863785.f940ad50107fe2c6e02a78c5100b2427. 2024-12-11T20:28:14,791 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bdd86b1c3693dd575903a0d36eeb4e01, disabling compactions & flushes 2024-12-11T20:28:14,791 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:28:14,791 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:28:14,791 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. after waiting 0 ms 2024-12-11T20:28:14,791 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:28:14,791 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bdd86b1c3693dd575903a0d36eeb4e01 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:14,815 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/.tmp/testFamily/7fe4e235a386492a9fa903b861a368df is 61, key is row-0/testFamily:testQualifier/1733948830218/Put/seqid=0 2024-12-11T20:28:14,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741859_1035 (size=5049) 2024-12-11T20:28:14,943 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 1588230740, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, bdd86b1c3693dd575903a0d36eeb4e01, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:15,116 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/ns/8b27e3cb56554de9a57bc7a487e4b84f 2024-12-11T20:28:15,142 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/table/af112e9f4fa84991ae97c0b8ad83bbbe is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733948828368/Put/seqid=0 2024-12-11T20:28:15,143 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-11T20:28:15,143 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1325): Online Regions={bdd86b1c3693dd575903a0d36eeb4e01=testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01., e3116e57515a6d989750370c39cbc844=testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., 0f077f99d334cd858db9d852d667e7a8=testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8., e6e1f0c5a512aa504f84ad541d42d8c2=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., fdc8ac339ad1b89b0b29e3734ea77e36=hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36., 1588230740=hbase:meta,,1.1588230740, 65cce501f100c8aafe4ecd17f541aefe=hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe., 4440fd1fba3c3512abafa8a681ee2568=testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568., 9680151e407195902f12fd73ae967fab=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.} 2024-12-11T20:28:15,143 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 1588230740, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, bdd86b1c3693dd575903a0d36eeb4e01, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:15,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741860_1036 (size=6699) 2024-12-11T20:28:15,149 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/table/af112e9f4fa84991ae97c0b8ad83bbbe 2024-12-11T20:28:15,160 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/info/3ed440b3b72e45998734e2b7f4b86a81 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/info/3ed440b3b72e45998734e2b7f4b86a81 2024-12-11T20:28:15,177 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/info/3ed440b3b72e45998734e2b7f4b86a81, entries=100, sequenceid=56, filesize=20.1 K 2024-12-11T20:28:15,183 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/ns/8b27e3cb56554de9a57bc7a487e4b84f as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/ns/8b27e3cb56554de9a57bc7a487e4b84f 2024-12-11T20:28:15,192 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/ns/8b27e3cb56554de9a57bc7a487e4b84f, entries=2, sequenceid=56, filesize=5.0 K 2024-12-11T20:28:15,194 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/.tmp/table/af112e9f4fa84991ae97c0b8ad83bbbe as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/table/af112e9f4fa84991ae97c0b8ad83bbbe 2024-12-11T20:28:15,206 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/table/af112e9f4fa84991ae97c0b8ad83bbbe, entries=20, sequenceid=56, filesize=6.5 K 2024-12-11T20:28:15,208 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1066ms, sequenceid=56, compaction requested=false 2024-12-11T20:28:15,220 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-11T20:28:15,221 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T20:28:15,221 INFO [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T20:28:15,221 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733948894141Running coprocessor pre-close hooks at 1733948894141Disabling compacts and flushes for region at 1733948894141Disabling writes for close at 1733948894142 (+1 ms)Obtaining lock to block concurrent updates at 1733948894144 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733948894144Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733948894152 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733948894154 (+2 ms)Flushing 1588230740/info: creating writer at 1733948894156 (+2 ms)Flushing 1588230740/info: appending metadata at 1733948894215 (+59 ms)Flushing 1588230740/info: closing flushed file at 1733948894215Flushing 1588230740/ns: creating writer at 1733948894681 (+466 ms)Flushing 1588230740/ns: appending metadata at 1733948894703 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733948894703Flushing 1588230740/table: creating writer at 1733948895124 (+421 ms)Flushing 1588230740/table: appending metadata at 1733948895141 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733948895142 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d88e03d: reopening flushed file at 1733948895158 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a0ddc8a: reopening flushed file at 1733948895177 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3df8e4d8: reopening flushed file at 1733948895193 (+16 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1066ms, sequenceid=56, compaction requested=false at 1733948895208 (+15 ms)Writing region close event to WAL at 1733948895210 (+2 ms)Running coprocessor post-close hooks at 1733948895220 (+10 ms)Closed at 1733948895221 (+1 ms) 2024-12-11T20:28:15,221 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T20:28:15,222 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/.tmp/testFamily/7fe4e235a386492a9fa903b861a368df 2024-12-11T20:28:15,236 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/.tmp/testFamily/7fe4e235a386492a9fa903b861a368df as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/testFamily/7fe4e235a386492a9fa903b861a368df 2024-12-11T20:28:15,245 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/testFamily/7fe4e235a386492a9fa903b861a368df, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:15,246 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bdd86b1c3693dd575903a0d36eeb4e01 in 455ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:15,252 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bdd86b1c3693dd575903a0d36eeb4e01/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:15,253 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:28:15,253 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bdd86b1c3693dd575903a0d36eeb4e01: Waiting for close lock at 1733948894791Running coprocessor pre-close hooks at 1733948894791Disabling compacts and flushes for region at 1733948894791Disabling writes for close at 1733948894791Obtaining lock to block concurrent updates at 1733948894791Preparing flush snapshotting stores in bdd86b1c3693dd575903a0d36eeb4e01 at 1733948894791Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948894792 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. at 1733948894792Flushing bdd86b1c3693dd575903a0d36eeb4e01/testFamily: creating writer at 1733948894792Flushing bdd86b1c3693dd575903a0d36eeb4e01/testFamily: appending metadata at 1733948894814 (+22 ms)Flushing bdd86b1c3693dd575903a0d36eeb4e01/testFamily: closing flushed file at 1733948894815 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24e65cde: reopening flushed file at 1733948895234 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bdd86b1c3693dd575903a0d36eeb4e01 in 455ms, sequenceid=8, compaction requested=false at 1733948895246 (+12 ms)Writing region close event to WAL at 1733948895247 (+1 ms)Running coprocessor post-close hooks at 1733948895253 (+6 ms)Closed at 1733948895253 2024-12-11T20:28:15,253 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733948827130.bdd86b1c3693dd575903a0d36eeb4e01. 2024-12-11T20:28:15,253 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e3116e57515a6d989750370c39cbc844, disabling compactions & flushes 2024-12-11T20:28:15,253 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:28:15,253 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:28:15,253 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. after waiting 0 ms 2024-12-11T20:28:15,253 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:28:15,254 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e3116e57515a6d989750370c39cbc844 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:15,272 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/.tmp/testFamily/9fa0d7a639624a94ba49b5dde96d6418 is 61, key is row-0/testFamily:testQualifier/1733948854654/Put/seqid=0 2024-12-11T20:28:15,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741861_1037 (size=5049) 2024-12-11T20:28:15,343 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:15,544 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e3116e57515a6d989750370c39cbc844, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:15,679 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/.tmp/testFamily/9fa0d7a639624a94ba49b5dde96d6418 2024-12-11T20:28:15,687 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/.tmp/testFamily/9fa0d7a639624a94ba49b5dde96d6418 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/testFamily/9fa0d7a639624a94ba49b5dde96d6418 2024-12-11T20:28:15,694 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/testFamily/9fa0d7a639624a94ba49b5dde96d6418, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:15,696 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e3116e57515a6d989750370c39cbc844 in 442ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:15,700 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScanner/e3116e57515a6d989750370c39cbc844/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:15,701 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:28:15,701 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e3116e57515a6d989750370c39cbc844: Waiting for close lock at 1733948895253Running coprocessor pre-close hooks at 1733948895253Disabling compacts and flushes for region at 1733948895253Disabling writes for close at 1733948895253Obtaining lock to block concurrent updates at 1733948895254 (+1 ms)Preparing flush snapshotting stores in e3116e57515a6d989750370c39cbc844 at 1733948895254Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948895254Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. at 1733948895255 (+1 ms)Flushing e3116e57515a6d989750370c39cbc844/testFamily: creating writer at 1733948895255Flushing e3116e57515a6d989750370c39cbc844/testFamily: appending metadata at 1733948895272 (+17 ms)Flushing e3116e57515a6d989750370c39cbc844/testFamily: closing flushed file at 1733948895272Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44e4917d: reopening flushed file at 1733948895686 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e3116e57515a6d989750370c39cbc844 in 442ms, sequenceid=8, compaction requested=false at 1733948895696 (+10 ms)Writing region close event to WAL at 1733948895696Running coprocessor post-close hooks at 1733948895701 (+5 ms)Closed at 1733948895701 2024-12-11T20:28:15,701 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733948851604.e3116e57515a6d989750370c39cbc844. 2024-12-11T20:28:15,701 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0f077f99d334cd858db9d852d667e7a8, disabling compactions & flushes 2024-12-11T20:28:15,701 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:28:15,701 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:28:15,701 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. after waiting 0 ms 2024-12-11T20:28:15,701 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:28:15,701 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0f077f99d334cd858db9d852d667e7a8 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:15,723 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/.tmp/testFamily/e94d79a430c04eeb8279f1b27ae9305f is 61, key is row-0/testFamily:testQualifier/1733948860743/Put/seqid=0 2024-12-11T20:28:15,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741862_1038 (size=5049) 2024-12-11T20:28:15,744 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:15,944 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:16,130 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/.tmp/testFamily/e94d79a430c04eeb8279f1b27ae9305f 2024-12-11T20:28:16,139 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/.tmp/testFamily/e94d79a430c04eeb8279f1b27ae9305f as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/testFamily/e94d79a430c04eeb8279f1b27ae9305f 2024-12-11T20:28:16,145 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-11T20:28:16,145 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1325): Online Regions={0f077f99d334cd858db9d852d667e7a8=testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8., e6e1f0c5a512aa504f84ad541d42d8c2=testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., fdc8ac339ad1b89b0b29e3734ea77e36=hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36., 65cce501f100c8aafe4ecd17f541aefe=hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe., 4440fd1fba3c3512abafa8a681ee2568=testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568., 9680151e407195902f12fd73ae967fab=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.} 2024-12-11T20:28:16,145 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 0f077f99d334cd858db9d852d667e7a8, 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:16,146 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/testFamily/e94d79a430c04eeb8279f1b27ae9305f, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:16,148 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0f077f99d334cd858db9d852d667e7a8 in 446ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:16,152 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnNextAsync/0f077f99d334cd858db9d852d667e7a8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:16,153 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:28:16,153 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0f077f99d334cd858db9d852d667e7a8: Waiting for close lock at 1733948895701Running coprocessor pre-close hooks at 1733948895701Disabling compacts and flushes for region at 1733948895701Disabling writes for close at 1733948895701Obtaining lock to block concurrent updates at 1733948895701Preparing flush snapshotting stores in 0f077f99d334cd858db9d852d667e7a8 at 1733948895701Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948895702 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. at 1733948895702Flushing 0f077f99d334cd858db9d852d667e7a8/testFamily: creating writer at 1733948895702Flushing 0f077f99d334cd858db9d852d667e7a8/testFamily: appending metadata at 1733948895723 (+21 ms)Flushing 0f077f99d334cd858db9d852d667e7a8/testFamily: closing flushed file at 1733948895723Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@105258f5: reopening flushed file at 1733948896138 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0f077f99d334cd858db9d852d667e7a8 in 446ms, sequenceid=8, compaction requested=false at 1733948896148 (+10 ms)Writing region close event to WAL at 1733948896148Running coprocessor post-close hooks at 1733948896153 (+5 ms)Closed at 1733948896153 2024-12-11T20:28:16,153 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733948857694.0f077f99d334cd858db9d852d667e7a8. 2024-12-11T20:28:16,153 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e6e1f0c5a512aa504f84ad541d42d8c2, disabling compactions & flushes 2024-12-11T20:28:16,153 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:16,153 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:16,153 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. after waiting 0 ms 2024-12-11T20:28:16,154 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:16,154 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e6e1f0c5a512aa504f84ad541d42d8c2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:16,171 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/.tmp/testFamily/0a0092f63cb2493c9dcc4370c68667b2 is 61, key is row-0/testFamily:testQualifier/1733948891097/Put/seqid=0 2024-12-11T20:28:16,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741863_1039 (size=5049) 2024-12-11T20:28:16,345 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:16,545 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab, e6e1f0c5a512aa504f84ad541d42d8c2, fdc8ac339ad1b89b0b29e3734ea77e36 2024-12-11T20:28:16,577 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/.tmp/testFamily/0a0092f63cb2493c9dcc4370c68667b2 2024-12-11T20:28:16,588 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/.tmp/testFamily/0a0092f63cb2493c9dcc4370c68667b2 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/testFamily/0a0092f63cb2493c9dcc4370c68667b2 2024-12-11T20:28:16,597 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/testFamily/0a0092f63cb2493c9dcc4370c68667b2, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:16,598 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e6e1f0c5a512aa504f84ad541d42d8c2 in 444ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:16,603 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testNormalScanTimeoutOnOpenScannerAsync/e6e1f0c5a512aa504f84ad541d42d8c2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:16,604 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:16,604 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e6e1f0c5a512aa504f84ad541d42d8c2: Waiting for close lock at 1733948896153Running coprocessor pre-close hooks at 1733948896153Disabling compacts and flushes for region at 1733948896153Disabling writes for close at 1733948896153Obtaining lock to block concurrent updates at 1733948896154 (+1 ms)Preparing flush snapshotting stores in e6e1f0c5a512aa504f84ad541d42d8c2 at 1733948896154Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948896154Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. at 1733948896154Flushing e6e1f0c5a512aa504f84ad541d42d8c2/testFamily: creating writer at 1733948896154Flushing e6e1f0c5a512aa504f84ad541d42d8c2/testFamily: appending metadata at 1733948896171 (+17 ms)Flushing e6e1f0c5a512aa504f84ad541d42d8c2/testFamily: closing flushed file at 1733948896171Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32b68ad4: reopening flushed file at 1733948896586 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e6e1f0c5a512aa504f84ad541d42d8c2 in 444ms, sequenceid=8, compaction requested=false at 1733948896598 (+12 ms)Writing region close event to WAL at 1733948896599 (+1 ms)Running coprocessor post-close hooks at 1733948896604 (+5 ms)Closed at 1733948896604 2024-12-11T20:28:16,604 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733948888049.e6e1f0c5a512aa504f84ad541d42d8c2. 2024-12-11T20:28:16,605 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fdc8ac339ad1b89b0b29e3734ea77e36, disabling compactions & flushes 2024-12-11T20:28:16,605 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:16,605 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:16,605 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. after waiting 0 ms 2024-12-11T20:28:16,605 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:16,605 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fdc8ac339ad1b89b0b29e3734ea77e36 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:16,627 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/.tmp/testFamily/0c012ad521334b1bb1e2dec883e5a149 is 61, key is row-0/testFamily:testQualifier/1733948882003/Put/seqid=0 2024-12-11T20:28:16,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741864_1040 (size=5049) 2024-12-11T20:28:16,633 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/.tmp/testFamily/0c012ad521334b1bb1e2dec883e5a149 2024-12-11T20:28:16,643 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/.tmp/testFamily/0c012ad521334b1bb1e2dec883e5a149 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/testFamily/0c012ad521334b1bb1e2dec883e5a149 2024-12-11T20:28:16,650 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/testFamily/0c012ad521334b1bb1e2dec883e5a149, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:16,652 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fdc8ac339ad1b89b0b29e3734ea77e36 in 47ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:16,658 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNext/fdc8ac339ad1b89b0b29e3734ea77e36/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:16,659 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:16,659 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fdc8ac339ad1b89b0b29e3734ea77e36: Waiting for close lock at 1733948896604Running coprocessor pre-close hooks at 1733948896604Disabling compacts and flushes for region at 1733948896604Disabling writes for close at 1733948896605 (+1 ms)Obtaining lock to block concurrent updates at 1733948896605Preparing flush snapshotting stores in fdc8ac339ad1b89b0b29e3734ea77e36 at 1733948896605Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948896605Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. at 1733948896606 (+1 ms)Flushing fdc8ac339ad1b89b0b29e3734ea77e36/testFamily: creating writer at 1733948896606Flushing fdc8ac339ad1b89b0b29e3734ea77e36/testFamily: appending metadata at 1733948896627 (+21 ms)Flushing fdc8ac339ad1b89b0b29e3734ea77e36/testFamily: closing flushed file at 1733948896627Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@493a59e2: reopening flushed file at 1733948896642 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fdc8ac339ad1b89b0b29e3734ea77e36 in 47ms, sequenceid=8, compaction requested=false at 1733948896652 (+10 ms)Writing region close event to WAL at 1733948896653 (+1 ms)Running coprocessor post-close hooks at 1733948896658 (+5 ms)Closed at 1733948896658 2024-12-11T20:28:16,659 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733948878946.fdc8ac339ad1b89b0b29e3734ea77e36. 2024-12-11T20:28:16,659 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65cce501f100c8aafe4ecd17f541aefe, disabling compactions & flushes 2024-12-11T20:28:16,659 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:28:16,659 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:28:16,659 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. after waiting 0 ms 2024-12-11T20:28:16,659 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:28:16,659 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 65cce501f100c8aafe4ecd17f541aefe 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:16,677 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/.tmp/testFamily/ba47a319700f41d19573cb574880d03a is 61, key is row-0/testFamily:testQualifier/1733948842474/Put/seqid=0 2024-12-11T20:28:16,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741865_1041 (size=5049) 2024-12-11T20:28:16,746 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab 2024-12-11T20:28:16,946 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 4440fd1fba3c3512abafa8a681ee2568, 65cce501f100c8aafe4ecd17f541aefe, 9680151e407195902f12fd73ae967fab 2024-12-11T20:28:17,083 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/.tmp/testFamily/ba47a319700f41d19573cb574880d03a 2024-12-11T20:28:17,093 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/.tmp/testFamily/ba47a319700f41d19573cb574880d03a as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/testFamily/ba47a319700f41d19573cb574880d03a 2024-12-11T20:28:17,101 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/testFamily/ba47a319700f41d19573cb574880d03a, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:17,102 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65cce501f100c8aafe4ecd17f541aefe in 443ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:17,107 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnNextAsync/65cce501f100c8aafe4ecd17f541aefe/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:17,108 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:28:17,108 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65cce501f100c8aafe4ecd17f541aefe: Waiting for close lock at 1733948896659Running coprocessor pre-close hooks at 1733948896659Disabling compacts and flushes for region at 1733948896659Disabling writes for close at 1733948896659Obtaining lock to block concurrent updates at 1733948896659Preparing flush snapshotting stores in 65cce501f100c8aafe4ecd17f541aefe at 1733948896659Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948896659Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. at 1733948896660 (+1 ms)Flushing 65cce501f100c8aafe4ecd17f541aefe/testFamily: creating writer at 1733948896660Flushing 65cce501f100c8aafe4ecd17f541aefe/testFamily: appending metadata at 1733948896677 (+17 ms)Flushing 65cce501f100c8aafe4ecd17f541aefe/testFamily: closing flushed file at 1733948896677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17c9f642: reopening flushed file at 1733948897092 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65cce501f100c8aafe4ecd17f541aefe in 443ms, sequenceid=8, compaction requested=false at 1733948897102 (+10 ms)Writing region close event to WAL at 1733948897103 (+1 ms)Running coprocessor post-close hooks at 1733948897108 (+5 ms)Closed at 1733948897108 2024-12-11T20:28:17,108 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733948839425.65cce501f100c8aafe4ecd17f541aefe. 2024-12-11T20:28:17,108 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4440fd1fba3c3512abafa8a681ee2568, disabling compactions & flushes 2024-12-11T20:28:17,108 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:28:17,108 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:28:17,108 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. after waiting 0 ms 2024-12-11T20:28:17,109 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:28:17,109 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4440fd1fba3c3512abafa8a681ee2568 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:17,130 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/.tmp/testFamily/a700a0d373c249228c7029a42fbfba22 is 61, key is row-0/testFamily:testQualifier/1733948851554/Put/seqid=0 2024-12-11T20:28:17,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741866_1042 (size=5049) 2024-12-11T20:28:17,145 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/.tmp/testFamily/a700a0d373c249228c7029a42fbfba22 2024-12-11T20:28:17,146 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-11T20:28:17,146 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1325): Online Regions={4440fd1fba3c3512abafa8a681ee2568=testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568., 9680151e407195902f12fd73ae967fab=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab.} 2024-12-11T20:28:17,146 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 4440fd1fba3c3512abafa8a681ee2568, 9680151e407195902f12fd73ae967fab 2024-12-11T20:28:17,156 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/.tmp/testFamily/a700a0d373c249228c7029a42fbfba22 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/testFamily/a700a0d373c249228c7029a42fbfba22 2024-12-11T20:28:17,166 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/testFamily/a700a0d373c249228c7029a42fbfba22, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:17,168 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4440fd1fba3c3512abafa8a681ee2568 in 59ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:17,176 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/default/testRetryOutOfOrderScannerNextException/4440fd1fba3c3512abafa8a681ee2568/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:17,178 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:28:17,178 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4440fd1fba3c3512abafa8a681ee2568: Waiting for close lock at 1733948897108Running coprocessor pre-close hooks at 1733948897108Disabling compacts and flushes for region at 1733948897108Disabling writes for close at 1733948897109 (+1 ms)Obtaining lock to block concurrent updates at 1733948897109Preparing flush snapshotting stores in 4440fd1fba3c3512abafa8a681ee2568 at 1733948897109Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948897109Flushing stores of testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. at 1733948897109Flushing 4440fd1fba3c3512abafa8a681ee2568/testFamily: creating writer at 1733948897109Flushing 4440fd1fba3c3512abafa8a681ee2568/testFamily: appending metadata at 1733948897130 (+21 ms)Flushing 4440fd1fba3c3512abafa8a681ee2568/testFamily: closing flushed file at 1733948897130Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69865ad9: reopening flushed file at 1733948897153 (+23 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4440fd1fba3c3512abafa8a681ee2568 in 59ms, sequenceid=8, compaction requested=false at 1733948897168 (+15 ms)Writing region close event to WAL at 1733948897168Running coprocessor post-close hooks at 1733948897177 (+9 ms)Closed at 1733948897177 2024-12-11T20:28:17,178 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733948848518.4440fd1fba3c3512abafa8a681ee2568. 2024-12-11T20:28:17,178 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9680151e407195902f12fd73ae967fab, disabling compactions & flushes 2024-12-11T20:28:17,178 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:28:17,178 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:28:17,178 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. after waiting 0 ms 2024-12-11T20:28:17,178 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:28:17,178 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9680151e407195902f12fd73ae967fab 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T20:28:17,204 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/.tmp/testFamily/acedf92b58bd489caab0adb8dd3ddbf3 is 61, key is row-0/testFamily:testQualifier/1733948872914/Put/seqid=0 2024-12-11T20:28:17,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741867_1043 (size=5049) 2024-12-11T20:28:17,346 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 9680151e407195902f12fd73ae967fab 2024-12-11T20:28:17,547 DEBUG [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1351): Waiting on 9680151e407195902f12fd73ae967fab 2024-12-11T20:28:17,611 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/.tmp/testFamily/acedf92b58bd489caab0adb8dd3ddbf3 2024-12-11T20:28:17,620 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/.tmp/testFamily/acedf92b58bd489caab0adb8dd3ddbf3 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/testFamily/acedf92b58bd489caab0adb8dd3ddbf3 2024-12-11T20:28:17,628 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/testFamily/acedf92b58bd489caab0adb8dd3ddbf3, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T20:28:17,629 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9680151e407195902f12fd73ae967fab in 451ms, sequenceid=8, compaction requested=false 2024-12-11T20:28:17,635 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9680151e407195902f12fd73ae967fab/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T20:28:17,636 INFO [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:28:17,636 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9680151e407195902f12fd73ae967fab: Waiting for close lock at 1733948897178Running coprocessor pre-close hooks at 1733948897178Disabling compacts and flushes for region at 1733948897178Disabling writes for close at 1733948897178Obtaining lock to block concurrent updates at 1733948897178Preparing flush snapshotting stores in 9680151e407195902f12fd73ae967fab at 1733948897178Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733948897179 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. at 1733948897180 (+1 ms)Flushing 9680151e407195902f12fd73ae967fab/testFamily: creating writer at 1733948897180Flushing 9680151e407195902f12fd73ae967fab/testFamily: appending metadata at 1733948897204 (+24 ms)Flushing 9680151e407195902f12fd73ae967fab/testFamily: closing flushed file at 1733948897204Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@610d1f87: reopening flushed file at 1733948897619 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9680151e407195902f12fd73ae967fab in 451ms, sequenceid=8, compaction requested=false at 1733948897629 (+10 ms)Writing region close event to WAL at 1733948897630 (+1 ms)Running coprocessor post-close hooks at 1733948897636 (+6 ms)Closed at 1733948897636 2024-12-11T20:28:17,637 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733948869865.9680151e407195902f12fd73ae967fab. 2024-12-11T20:28:17,747 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,39739,1733948793470; all regions closed. 2024-12-11T20:28:17,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741834_1010 (size=19479) 2024-12-11T20:28:17,758 DEBUG [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/oldWALs 2024-12-11T20:28:17,758 INFO [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9fe0640122ec%2C39739%2C1733948793470.meta:.meta(num 1733948826741) 2024-12-11T20:28:17,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741833_1009 (size=20031) 2024-12-11T20:28:17,765 DEBUG [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/oldWALs 2024-12-11T20:28:17,765 INFO [RS:0;9fe0640122ec:39739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9fe0640122ec%2C39739%2C1733948793470:(num 1733948826300) 2024-12-11T20:28:17,765 DEBUG [RS:0;9fe0640122ec:39739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:28:17,765 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T20:28:17,765 INFO [RS:0;9fe0640122ec:39739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T20:28:17,765 INFO [RS:0;9fe0640122ec:39739 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T20:28:17,765 INFO [RS:0;9fe0640122ec:39739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T20:28:17,765 INFO [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T20:28:17,767 INFO [RS:0;9fe0640122ec:39739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39739 2024-12-11T20:28:17,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,39739,1733948793470 2024-12-11T20:28:17,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T20:28:17,771 INFO [RS:0;9fe0640122ec:39739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T20:28:17,773 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,39739,1733948793470] 2024-12-11T20:28:17,775 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,39739,1733948793470 already deleted, retry=false 2024-12-11T20:28:17,775 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,39739,1733948793470 expired; onlineServers=0 2024-12-11T20:28:17,776 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,43275,1733948792764' ***** 2024-12-11T20:28:17,776 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T20:28:17,776 INFO [M:0;9fe0640122ec:43275 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T20:28:17,776 INFO [M:0;9fe0640122ec:43275 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T20:28:17,776 DEBUG [M:0;9fe0640122ec:43275 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T20:28:17,776 DEBUG [M:0;9fe0640122ec:43275 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T20:28:17,776 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T20:28:17,776 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948796109 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948796109,5,FailOnTimeoutGroup] 2024-12-11T20:28:17,776 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948796106 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948796106,5,FailOnTimeoutGroup] 2024-12-11T20:28:17,776 INFO [M:0;9fe0640122ec:43275 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T20:28:17,777 INFO [M:0;9fe0640122ec:43275 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T20:28:17,777 DEBUG [M:0;9fe0640122ec:43275 {}] master.HMaster(1795): Stopping service threads 2024-12-11T20:28:17,777 INFO [M:0;9fe0640122ec:43275 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T20:28:17,777 INFO [M:0;9fe0640122ec:43275 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T20:28:17,778 INFO [M:0;9fe0640122ec:43275 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T20:28:17,778 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T20:28:17,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T20:28:17,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:28:17,779 DEBUG [M:0;9fe0640122ec:43275 {}] zookeeper.ZKUtil(347): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-11T20:28:17,779 WARN [M:0;9fe0640122ec:43275 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-11T20:28:17,780 INFO [M:0;9fe0640122ec:43275 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/.lastflushedseqids 2024-12-11T20:28:17,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741868_1044 (size=649) 2024-12-11T20:28:17,817 INFO [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T20:28:17,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:28:17,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39739-0x100cb04ee870001, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:28:17,874 INFO [RS:0;9fe0640122ec:39739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T20:28:17,875 INFO [RS:0;9fe0640122ec:39739 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,39739,1733948793470; zookeeper connection closed. 2024-12-11T20:28:17,875 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38e47eff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38e47eff 2024-12-11T20:28:17,876 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-11T20:28:18,194 INFO [M:0;9fe0640122ec:43275 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T20:28:18,195 INFO [M:0;9fe0640122ec:43275 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T20:28:18,195 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T20:28:18,195 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:28:18,195 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:28:18,195 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T20:28:18,195 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:28:18,195 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.96 KB heapSize=190.83 KB 2024-12-11T20:28:18,213 DEBUG [M:0;9fe0640122ec:43275 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e465f8c2f30e49b5a90ed6a0193ae26b is 82, key is hbase:meta,,1/info:regioninfo/1733948826829/Put/seqid=0 2024-12-11T20:28:18,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741869_1045 (size=5672) 2024-12-11T20:28:18,619 INFO [M:0;9fe0640122ec:43275 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e465f8c2f30e49b5a90ed6a0193ae26b 2024-12-11T20:28:18,646 DEBUG [M:0;9fe0640122ec:43275 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8a2cc8bd5c6f406bbe816e7f9e64ab05 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733948828375/Put/seqid=0 2024-12-11T20:28:18,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741870_1046 (size=14067) 2024-12-11T20:28:19,052 INFO [M:0;9fe0640122ec:43275 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.40 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8a2cc8bd5c6f406bbe816e7f9e64ab05 2024-12-11T20:28:19,060 INFO [M:0;9fe0640122ec:43275 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a2cc8bd5c6f406bbe816e7f9e64ab05 2024-12-11T20:28:19,080 DEBUG [M:0;9fe0640122ec:43275 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c32fce4872654960ab48f8acddcfaeea is 69, key is 9fe0640122ec,39739,1733948793470/rs:state/1733948796160/Put/seqid=0 2024-12-11T20:28:19,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741871_1047 (size=5156) 2024-12-11T20:28:19,486 INFO [M:0;9fe0640122ec:43275 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c32fce4872654960ab48f8acddcfaeea 2024-12-11T20:28:19,494 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e465f8c2f30e49b5a90ed6a0193ae26b as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e465f8c2f30e49b5a90ed6a0193ae26b 2024-12-11T20:28:19,502 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e465f8c2f30e49b5a90ed6a0193ae26b, entries=8, sequenceid=317, filesize=5.5 K 2024-12-11T20:28:19,504 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8a2cc8bd5c6f406bbe816e7f9e64ab05 as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8a2cc8bd5c6f406bbe816e7f9e64ab05 2024-12-11T20:28:19,510 INFO [M:0;9fe0640122ec:43275 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a2cc8bd5c6f406bbe816e7f9e64ab05 2024-12-11T20:28:19,510 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8a2cc8bd5c6f406bbe816e7f9e64ab05, entries=33, sequenceid=317, filesize=13.7 K 2024-12-11T20:28:19,512 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c32fce4872654960ab48f8acddcfaeea as hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c32fce4872654960ab48f8acddcfaeea 2024-12-11T20:28:19,518 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44245/user/jenkins/test-data/570a2835-a7e1-c7b5-9138-a2590bff9c21/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c32fce4872654960ab48f8acddcfaeea, entries=1, sequenceid=317, filesize=5.0 K 2024-12-11T20:28:19,520 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.96 KB/166867, heapSize ~190.53 KB/195104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1325ms, sequenceid=317, compaction requested=false 2024-12-11T20:28:19,521 INFO [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:28:19,522 DEBUG [M:0;9fe0640122ec:43275 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733948898195Disabling compacts and flushes for region at 1733948898195Disabling writes for close at 1733948898195Obtaining lock to block concurrent updates at 1733948898195Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733948898195Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166867, getHeapSize=195344, getOffHeapSize=0, getCellsCount=364 at 1733948898196 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733948898196Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733948898196Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733948898213 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733948898213Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733948898626 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733948898645 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733948898645Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733948899060 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733948899080 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733948899080Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ca61a4a: reopening flushed file at 1733948899493 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c9b37b: reopening flushed file at 1733948899503 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46f0acc1: reopening flushed file at 1733948899511 (+8 ms)Finished flush of dataSize ~162.96 KB/166867, heapSize ~190.53 KB/195104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1325ms, sequenceid=317, compaction requested=false at 1733948899520 (+9 ms)Writing region close event to WAL at 1733948899521 (+1 ms)Closed at 1733948899521 2024-12-11T20:28:19,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42745 is added to blk_1073741830_1006 (size=188079) 2024-12-11T20:28:19,525 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T20:28:19,525 INFO [M:0;9fe0640122ec:43275 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T20:28:19,526 INFO [M:0;9fe0640122ec:43275 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43275 2024-12-11T20:28:19,526 INFO [M:0;9fe0640122ec:43275 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T20:28:19,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:28:19,628 INFO [M:0;9fe0640122ec:43275 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T20:28:19,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43275-0x100cb04ee870000, quorum=127.0.0.1:59124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:28:19,633 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T20:28:19,635 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T20:28:19,635 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T20:28:19,635 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T20:28:19,635 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/hadoop.log.dir/,STOPPED} 2024-12-11T20:28:19,638 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T20:28:19,637 WARN [BP-555049335-172.17.0.2-1733948789681 heartbeating to localhost/127.0.0.1:44245 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T20:28:19,638 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T20:28:19,638 WARN [BP-555049335-172.17.0.2-1733948789681 heartbeating to localhost/127.0.0.1:44245 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-555049335-172.17.0.2-1733948789681 (Datanode Uuid a62d8e52-e4a9-4ea7-9c7a-23dd2eff4ff3) service to localhost/127.0.0.1:44245 2024-12-11T20:28:19,640 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/data/data1/current/BP-555049335-172.17.0.2-1733948789681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T20:28:19,640 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/cluster_d7793308-fcb7-8efe-53c5-5c8e64742881/data/data2/current/BP-555049335-172.17.0.2-1733948789681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T20:28:19,641 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T20:28:19,649 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T20:28:19,649 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T20:28:19,650 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T20:28:19,650 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T20:28:19,650 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d00b24f8-13db-4d92-09e3-e17ec92dc052/hadoop.log.dir/,STOPPED} 2024-12-11T20:28:19,660 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T20:28:19,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down