2024-11-23 15:55:29,322 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-23 15:55:29,343 main DEBUG Took 0.019075 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 15:55:29,343 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 15:55:29,344 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 15:55:29,345 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 15:55:29,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,355 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 15:55:29,372 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,375 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,377 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,378 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,380 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,381 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,382 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,383 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,385 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,386 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,387 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:55:29,388 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,388 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 15:55:29,390 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:55:29,392 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 15:55:29,395 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 15:55:29,396 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 15:55:29,398 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 15:55:29,398 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 15:55:29,409 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 15:55:29,412 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 15:55:29,415 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 15:55:29,415 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 15:55:29,416 main DEBUG createAppenders(={Console}) 2024-11-23 15:55:29,417 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-23 15:55:29,417 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-23 15:55:29,417 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-23 15:55:29,418 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 15:55:29,418 main DEBUG OutputStream closed 2024-11-23 15:55:29,419 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 15:55:29,419 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 15:55:29,419 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-23 15:55:29,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 15:55:29,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 15:55:29,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 15:55:29,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 15:55:29,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 15:55:29,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 15:55:29,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 15:55:29,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 15:55:29,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 15:55:29,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 15:55:29,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 15:55:29,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 15:55:29,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 15:55:29,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 15:55:29,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 15:55:29,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 15:55:29,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 15:55:29,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 15:55:29,527 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 15:55:29,528 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-23 15:55:29,528 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 15:55:29,529 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-23T15:55:29,546 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-23 15:55:29,549 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 15:55:29,549 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T15:55:29,781 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5 2024-11-23T15:55:29,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T15:55:29,822 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9, deleteOnExit=true 2024-11-23T15:55:29,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T15:55:29,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/test.cache.data in system properties and HBase conf 2024-11-23T15:55:29,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T15:55:29,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/hadoop.log.dir in system properties and HBase conf 2024-11-23T15:55:29,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T15:55:29,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T15:55:29,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T15:55:29,920 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T15:55:30,014 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T15:55:30,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T15:55:30,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T15:55:30,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T15:55:30,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T15:55:30,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T15:55:30,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T15:55:30,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T15:55:30,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T15:55:30,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T15:55:30,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/nfs.dump.dir in system properties and HBase conf 2024-11-23T15:55:30,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/java.io.tmpdir in system properties and HBase conf 2024-11-23T15:55:30,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T15:55:30,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T15:55:30,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T15:55:31,121 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T15:55:31,191 INFO [Time-limited test {}] log.Log(170): Logging initialized @2564ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T15:55:31,257 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:55:31,320 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:55:31,350 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:55:31,350 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:55:31,352 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T15:55:31,366 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:55:31,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:55:31,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:55:31,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@562f0d0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/java.io.tmpdir/jetty-localhost-42761-hadoop-hdfs-3_4_1-tests_jar-_-any-7164164047177697634/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T15:55:31,579 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:42761} 2024-11-23T15:55:31,581 INFO [Time-limited test {}] server.Server(415): Started @2954ms 2024-11-23T15:55:32,070 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:55:32,077 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:55:32,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:55:32,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:55:32,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T15:55:32,079 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:55:32,080 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:55:32,178 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc78d53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/java.io.tmpdir/jetty-localhost-39507-hadoop-hdfs-3_4_1-tests_jar-_-any-5359210015033410244/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:55:32,179 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:39507} 2024-11-23T15:55:32,179 INFO [Time-limited test {}] server.Server(415): Started @3554ms 2024-11-23T15:55:32,227 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T15:55:33,062 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/data/data2/current/BP-942193177-172.17.0.2-1732377330606/current, will proceed with Du for space computation calculation, 2024-11-23T15:55:33,062 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/data/data1/current/BP-942193177-172.17.0.2-1732377330606/current, will proceed with Du for space computation calculation, 2024-11-23T15:55:33,089 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T15:55:33,146 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9658d7e50c1d3979 with lease ID 0x913c561424d2b526: Processing first storage report for DS-06c1e197-db8b-4296-b450-5445c0de52b1 from datanode DatanodeRegistration(127.0.0.1:32865, datanodeUuid=19145e33-d8d6-459d-8a4a-15be26ca102e, infoPort=42941, infoSecurePort=0, ipcPort=36951, storageInfo=lv=-57;cid=testClusterID;nsid=1360910228;c=1732377330606) 2024-11-23T15:55:33,147 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9658d7e50c1d3979 with lease ID 0x913c561424d2b526: from storage DS-06c1e197-db8b-4296-b450-5445c0de52b1 node DatanodeRegistration(127.0.0.1:32865, datanodeUuid=19145e33-d8d6-459d-8a4a-15be26ca102e, infoPort=42941, infoSecurePort=0, ipcPort=36951, storageInfo=lv=-57;cid=testClusterID;nsid=1360910228;c=1732377330606), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T15:55:33,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9658d7e50c1d3979 with lease ID 0x913c561424d2b526: Processing first storage report for DS-408720f0-5eb1-4c09-894a-4cbaf13cc57f from datanode DatanodeRegistration(127.0.0.1:32865, datanodeUuid=19145e33-d8d6-459d-8a4a-15be26ca102e, infoPort=42941, infoSecurePort=0, ipcPort=36951, storageInfo=lv=-57;cid=testClusterID;nsid=1360910228;c=1732377330606) 2024-11-23T15:55:33,147 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9658d7e50c1d3979 with lease ID 0x913c561424d2b526: from storage DS-408720f0-5eb1-4c09-894a-4cbaf13cc57f node DatanodeRegistration(127.0.0.1:32865, datanodeUuid=19145e33-d8d6-459d-8a4a-15be26ca102e, infoPort=42941, infoSecurePort=0, ipcPort=36951, storageInfo=lv=-57;cid=testClusterID;nsid=1360910228;c=1732377330606), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T15:55:33,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5 2024-11-23T15:55:33,289 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/zookeeper_0, clientPort=52376, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T15:55:33,298 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52376 2024-11-23T15:55:33,313 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:33,317 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:33,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741825_1001 (size=7) 2024-11-23T15:55:34,023 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 with version=8 2024-11-23T15:55:34,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/hbase-staging 2024-11-23T15:55:34,110 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T15:55:34,303 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b712f9af2c12:0 server-side Connection retries=9 2024-11-23T15:55:34,312 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:55:34,312 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:55:34,316 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:55:34,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:55:34,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:55:34,454 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T15:55:34,520 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T15:55:34,528 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T15:55:34,531 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:55:34,556 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 97580 (auto-detected) 2024-11-23T15:55:34,557 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T15:55:34,577 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34557 2024-11-23T15:55:34,599 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34557 connecting to ZooKeeper ensemble=127.0.0.1:52376 2024-11-23T15:55:34,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:345570x0, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:55:34,668 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34557-0x10169b1e0db0000 connected 2024-11-23T15:55:34,740 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:34,742 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:34,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:55:34,757 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61, hbase.cluster.distributed=false 2024-11-23T15:55:34,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:55:34,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34557 2024-11-23T15:55:34,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34557 2024-11-23T15:55:34,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34557 2024-11-23T15:55:34,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34557 2024-11-23T15:55:34,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34557 2024-11-23T15:55:34,892 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b712f9af2c12:0 server-side Connection retries=9 2024-11-23T15:55:34,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:55:34,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:55:34,893 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:55:34,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:55:34,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:55:34,896 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T15:55:34,899 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:55:34,900 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41227 2024-11-23T15:55:34,902 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41227 connecting to ZooKeeper ensemble=127.0.0.1:52376 2024-11-23T15:55:34,903 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:34,906 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:34,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412270x0, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:55:34,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412270x0, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:55:34,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41227-0x10169b1e0db0001 connected 2024-11-23T15:55:34,934 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T15:55:34,941 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T15:55:34,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T15:55:34,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:55:34,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41227 2024-11-23T15:55:34,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41227 2024-11-23T15:55:34,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41227 2024-11-23T15:55:34,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41227 2024-11-23T15:55:34,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41227 2024-11-23T15:55:34,966 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b712f9af2c12:34557 2024-11-23T15:55:34,967 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b712f9af2c12,34557,1732377334154 2024-11-23T15:55:34,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:55:34,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:55:34,982 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b712f9af2c12,34557,1732377334154 2024-11-23T15:55:35,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T15:55:35,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:35,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:35,014 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T15:55:35,016 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b712f9af2c12,34557,1732377334154 from backup master directory 2024-11-23T15:55:35,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:55:35,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b712f9af2c12,34557,1732377334154 2024-11-23T15:55:35,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:55:35,040 WARN [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:55:35,041 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b712f9af2c12,34557,1732377334154 2024-11-23T15:55:35,043 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T15:55:35,045 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T15:55:35,098 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/hbase.id] with ID: e52858c2-6096-4906-8ceb-18f6d3f3a43d 2024-11-23T15:55:35,098 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/.tmp/hbase.id 2024-11-23T15:55:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741826_1002 (size=42) 2024-11-23T15:55:35,511 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/.tmp/hbase.id]:[hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/hbase.id] 2024-11-23T15:55:35,560 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:55:35,566 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T15:55:35,584 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-23T15:55:35,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:35,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:35,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741827_1003 (size=196) 2024-11-23T15:55:36,055 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:55:36,057 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T15:55:36,070 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:55:36,073 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:55:36,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741828_1004 (size=1189) 2024-11-23T15:55:36,526 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store 2024-11-23T15:55:36,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741829_1005 (size=34) 2024-11-23T15:55:36,955 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T15:55:36,961 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:55:36,963 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T15:55:36,963 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:55:36,963 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:55:36,965 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T15:55:36,965 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:55:36,965 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:55:36,966 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732377336963Disabling compacts and flushes for region at 1732377336963Disabling writes for close at 1732377336965 (+2 ms)Writing region close event to WAL at 1732377336965Closed at 1732377336965 2024-11-23T15:55:36,968 WARN [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/.initializing 2024-11-23T15:55:36,968 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/WALs/b712f9af2c12,34557,1732377334154 2024-11-23T15:55:36,976 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:55:36,989 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C34557%2C1732377334154, suffix=, logDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/WALs/b712f9af2c12,34557,1732377334154, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/oldWALs, maxLogs=10 2024-11-23T15:55:37,010 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/WALs/b712f9af2c12,34557,1732377334154/b712f9af2c12%2C34557%2C1732377334154.1732377336994, exclude list is [], retry=0 2024-11-23T15:55:37,024 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32865,DS-06c1e197-db8b-4296-b450-5445c0de52b1,DISK] 2024-11-23T15:55:37,027 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T15:55:37,059 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/WALs/b712f9af2c12,34557,1732377334154/b712f9af2c12%2C34557%2C1732377334154.1732377336994 2024-11-23T15:55:37,060 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42941:42941)] 2024-11-23T15:55:37,061 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:55:37,061 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:55:37,065 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,066 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T15:55:37,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:37,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:55:37,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,132 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T15:55:37,132 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:37,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:55:37,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T15:55:37,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:37,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:55:37,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T15:55:37,141 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:37,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:55:37,142 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,146 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,147 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,152 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,153 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,157 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:55:37,161 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:55:37,166 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:55:37,167 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67851918, jitterRate=0.011072367429733276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:55:37,174 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732377337077Initializing all the Stores at 1732377337079 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377337080 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377337080Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377337081 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377337081Cleaning up temporary data from old regions at 1732377337153 (+72 ms)Region opened successfully at 1732377337174 (+21 ms) 2024-11-23T15:55:37,176 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T15:55:37,204 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170bdcf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:55:37,231 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T15:55:37,241 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T15:55:37,241 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T15:55:37,243 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T15:55:37,245 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T15:55:37,249 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T15:55:37,249 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T15:55:37,271 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T15:55:37,279 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T15:55:37,338 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T15:55:37,341 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T15:55:37,344 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T15:55:37,354 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T15:55:37,357 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T15:55:37,362 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T15:55:37,370 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T15:55:37,372 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T15:55:37,379 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T15:55:37,402 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T15:55:37,412 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T15:55:37,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:55:37,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:55:37,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:37,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:37,425 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b712f9af2c12,34557,1732377334154, sessionid=0x10169b1e0db0000, setting cluster-up flag (Was=false) 2024-11-23T15:55:37,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:37,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:37,479 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T15:55:37,482 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b712f9af2c12,34557,1732377334154 2024-11-23T15:55:37,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:37,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:55:37,529 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T15:55:37,531 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b712f9af2c12,34557,1732377334154 2024-11-23T15:55:37,540 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T15:55:37,557 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(746): ClusterId : e52858c2-6096-4906-8ceb-18f6d3f3a43d 2024-11-23T15:55:37,559 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T15:55:37,573 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T15:55:37,573 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T15:55:37,581 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T15:55:37,581 DEBUG [RS:0;b712f9af2c12:41227 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b037dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:55:37,594 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b712f9af2c12:41227 2024-11-23T15:55:37,597 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T15:55:37,597 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T15:55:37,597 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T15:55:37,599 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(2659): reportForDuty to master=b712f9af2c12,34557,1732377334154 with port=41227, startcode=1732377334860 2024-11-23T15:55:37,608 DEBUG [RS:0;b712f9af2c12:41227 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T15:55:37,608 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T15:55:37,616 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T15:55:37,622 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T15:55:37,627 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b712f9af2c12,34557,1732377334154 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T15:55:37,633 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:55:37,634 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:55:37,634 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:55:37,634 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:55:37,634 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b712f9af2c12:0, corePoolSize=10, maxPoolSize=10 2024-11-23T15:55:37,634 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:55:37,635 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:55:37,635 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:55:37,636 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732377367636 2024-11-23T15:55:37,638 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T15:55:37,640 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T15:55:37,640 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:55:37,641 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T15:55:37,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58211, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T15:55:37,643 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T15:55:37,643 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T15:55:37,644 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T15:55:37,644 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T15:55:37,646 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:37,646 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T15:55:37,646 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T15:55:37,648 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34557 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T15:55:37,650 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T15:55:37,652 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T15:55:37,652 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T15:55:37,654 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T15:55:37,655 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T15:55:37,657 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732377337656,5,FailOnTimeoutGroup] 2024-11-23T15:55:37,657 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732377337657,5,FailOnTimeoutGroup] 2024-11-23T15:55:37,657 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T15:55:37,658 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T15:55:37,659 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T15:55:37,659 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T15:55:37,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741831_1007 (size=1321) 2024-11-23T15:55:37,674 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T15:55:37,674 WARN [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-23T15:55:38,067 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T15:55:38,069 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:55:38,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741832_1008 (size=32) 2024-11-23T15:55:38,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:55:38,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:55:38,488 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:55:38,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:38,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:55:38,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:55:38,493 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:55:38,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:38,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:55:38,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:55:38,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:55:38,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:38,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:55:38,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:55:38,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:55:38,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:55:38,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:55:38,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:55:38,504 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740 2024-11-23T15:55:38,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740 2024-11-23T15:55:38,508 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:55:38,508 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:55:38,509 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:55:38,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:55:38,514 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:55:38,515 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73836278, jitterRate=0.10024628043174744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:55:38,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732377338482Initializing all the Stores at 1732377338484 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377338484Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377338484Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377338484Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377338485 (+1 ms)Cleaning up temporary data from old regions at 1732377338508 (+23 ms)Region opened successfully at 1732377338519 (+11 ms) 2024-11-23T15:55:38,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:55:38,519 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:55:38,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:55:38,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:55:38,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:55:38,521 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:55:38,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732377338519Disabling compacts and flushes for region at 1732377338519Disabling writes for close at 1732377338519Writing region close event to WAL at 1732377338520 (+1 ms)Closed at 1732377338521 (+1 ms) 2024-11-23T15:55:38,524 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:55:38,524 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T15:55:38,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T15:55:38,535 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:55:38,538 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T15:55:38,689 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:39,690 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:40,690 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:41,691 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:42,692 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:43,692 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:44,517 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T15:55:44,517 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T15:55:44,693 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:45,694 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:46,695 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:47,696 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:48,696 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:49,699 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:50,699 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:51,700 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:52,701 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:53,702 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:54,703 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:55,704 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:56,705 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:57,706 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:58,707 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:55:59,708 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:00,709 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:01,710 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:02,711 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:03,218 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:56:03,712 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:04,713 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:05,714 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:06,715 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:07,676 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(2659): reportForDuty to master=b712f9af2c12,34557,1732377334154 with port=41227, startcode=1732377334860 2024-11-23T15:56:07,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34557 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b712f9af2c12,41227,1732377334860 2024-11-23T15:56:07,682 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34557 {}] master.ServerManager(517): Registering regionserver=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:07,692 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:56:07,692 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39973 2024-11-23T15:56:07,692 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T15:56:07,716 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:07,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:56:07,743 DEBUG [RS:0;b712f9af2c12:41227 {}] zookeeper.ZKUtil(111): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b712f9af2c12,41227,1732377334860 2024-11-23T15:56:07,743 WARN [RS:0;b712f9af2c12:41227 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:56:07,744 INFO [RS:0;b712f9af2c12:41227 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:56:07,744 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860 2024-11-23T15:56:07,748 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b712f9af2c12,41227,1732377334860] 2024-11-23T15:56:07,769 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T15:56:07,779 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T15:56:07,784 INFO [RS:0;b712f9af2c12:41227 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:56:07,784 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,785 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T15:56:07,789 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T15:56:07,791 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,791 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,791 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,791 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,791 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,791 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,792 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:56:07,793 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:56:07,793 DEBUG [RS:0;b712f9af2c12:41227 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:56:07,793 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,794 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,794 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,794 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,794 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,794 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,41227,1732377334860-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:56:07,817 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T15:56:07,819 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,41227,1732377334860-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,819 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,819 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.Replication(171): b712f9af2c12,41227,1732377334860 started 2024-11-23T15:56:07,838 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:07,838 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1482): Serving as b712f9af2c12,41227,1732377334860, RpcServer on b712f9af2c12/172.17.0.2:41227, sessionid=0x10169b1e0db0001 2024-11-23T15:56:07,839 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T15:56:07,839 DEBUG [RS:0;b712f9af2c12:41227 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b712f9af2c12,41227,1732377334860 2024-11-23T15:56:07,840 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,41227,1732377334860' 2024-11-23T15:56:07,840 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T15:56:07,841 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T15:56:07,842 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T15:56:07,842 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T15:56:07,842 DEBUG [RS:0;b712f9af2c12:41227 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b712f9af2c12,41227,1732377334860 2024-11-23T15:56:07,842 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,41227,1732377334860' 2024-11-23T15:56:07,842 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T15:56:07,843 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T15:56:07,844 DEBUG [RS:0;b712f9af2c12:41227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T15:56:07,844 INFO [RS:0;b712f9af2c12:41227 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T15:56:07,844 INFO [RS:0;b712f9af2c12:41227 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T15:56:08,717 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:09,718 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:10,719 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:11,720 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:12,721 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:13,722 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:14,723 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:15,724 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:16,725 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:17,726 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:18,727 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:19,729 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:20,730 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:21,731 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:22,732 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:23,733 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:24,734 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:25,735 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:26,736 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:27,737 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:28,738 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:29,739 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:30,740 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:31,741 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:32,742 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:33,218 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:56:33,743 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:34,745 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:35,746 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:36,747 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:37,747 WARN [b712f9af2c12:34557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:56:37,856 INFO [RS:0;b712f9af2c12:41227 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:56:37,860 INFO [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C41227%2C1732377334860, suffix=, logDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/oldWALs, maxLogs=32 2024-11-23T15:56:37,875 DEBUG [RS:0;b712f9af2c12:41227 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860/b712f9af2c12%2C41227%2C1732377334860.1732377397862, exclude list is [], retry=0 2024-11-23T15:56:37,879 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32865,DS-06c1e197-db8b-4296-b450-5445c0de52b1,DISK] 2024-11-23T15:56:37,882 INFO [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860/b712f9af2c12%2C41227%2C1732377334860.1732377397862 2024-11-23T15:56:37,883 DEBUG [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42941:42941)] 2024-11-23T15:56:38,002 DEBUG [b712f9af2c12:34557 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T15:56:38,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:38,024 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,41227,1732377334860, state=OPENING 2024-11-23T15:56:38,054 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T15:56:38,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:56:38,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:56:38,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:56:38,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:56:38,069 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:56:38,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:56:38,251 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:56:38,254 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53197, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:56:38,264 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T15:56:38,264 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:56:38,265 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T15:56:38,268 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C41227%2C1732377334860.meta, suffix=.meta, logDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/oldWALs, maxLogs=32 2024-11-23T15:56:38,285 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860/b712f9af2c12%2C41227%2C1732377334860.meta.1732377398270.meta, exclude list is [], retry=0 2024-11-23T15:56:38,288 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32865,DS-06c1e197-db8b-4296-b450-5445c0de52b1,DISK] 2024-11-23T15:56:38,291 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/WALs/b712f9af2c12,41227,1732377334860/b712f9af2c12%2C41227%2C1732377334860.meta.1732377398270.meta 2024-11-23T15:56:38,292 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42941:42941)] 2024-11-23T15:56:38,292 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:56:38,294 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T15:56:38,296 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T15:56:38,300 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T15:56:38,304 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T15:56:38,305 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:38,305 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T15:56:38,305 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T15:56:38,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:56:38,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:56:38,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:38,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:56:38,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:56:38,314 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:56:38,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:38,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:56:38,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:56:38,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:56:38,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:38,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:56:38,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:56:38,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:56:38,323 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:38,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:56:38,324 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:56:38,325 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740 2024-11-23T15:56:38,327 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740 2024-11-23T15:56:38,330 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:56:38,330 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:56:38,331 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:56:38,333 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:56:38,334 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64575583, jitterRate=-0.037748828530311584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:56:38,334 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T15:56:38,336 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732377398305Writing region info on filesystem at 1732377398306 (+1 ms)Initializing all the Stores at 1732377398307 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377398308 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377398308Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377398308Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732377398308Cleaning up temporary data from old regions at 1732377398330 (+22 ms)Running coprocessor post-open hooks at 1732377398335 (+5 ms)Region opened successfully at 1732377398336 (+1 ms) 2024-11-23T15:56:38,342 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732377398244 2024-11-23T15:56:38,352 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T15:56:38,353 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T15:56:38,354 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:38,357 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,41227,1732377334860, state=OPEN 2024-11-23T15:56:38,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:56:38,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:56:38,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:56:38,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:56:38,429 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:38,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T15:56:38,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,41227,1732377334860 in 358 msec 2024-11-23T15:56:38,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T15:56:38,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 59.9130 sec 2024-11-23T15:56:38,449 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:56:38,449 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T15:56:38,467 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:56:38,468 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,41227,1732377334860, seqNum=-1] 2024-11-23T15:56:38,486 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:56:38,488 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:56:38,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 0.937 sec 2024-11-23T15:56:38,508 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732377398508, completionTime=-1 2024-11-23T15:56:38,511 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T15:56:38,512 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T15:56:38,542 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T15:56:38,542 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732377458542 2024-11-23T15:56:38,542 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732377518542 2024-11-23T15:56:38,542 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-23T15:56:38,543 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34557,1732377334154-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,544 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34557,1732377334154-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,544 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34557,1732377334154-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,545 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b712f9af2c12:34557, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,545 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,546 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,551 DEBUG [master/b712f9af2c12:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T15:56:38,571 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.530sec 2024-11-23T15:56:38,572 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T15:56:38,573 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T15:56:38,574 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T15:56:38,574 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T15:56:38,574 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T15:56:38,575 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34557,1732377334154-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:56:38,575 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34557,1732377334154-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T15:56:38,583 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T15:56:38,583 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T15:56:38,584 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34557,1732377334154-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:56:38,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5e818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:56:38,649 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T15:56:38,649 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T15:56:38,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,34557,-1 for getting cluster id 2024-11-23T15:56:38,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:56:38,663 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e52858c2-6096-4906-8ceb-18f6d3f3a43d' 2024-11-23T15:56:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:56:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e52858c2-6096-4906-8ceb-18f6d3f3a43d" 2024-11-23T15:56:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4411660c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:56:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,34557,-1] 2024-11-23T15:56:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:56:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:56:38,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:56:38,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ba9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:56:38,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:56:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,41227,1732377334860, seqNum=-1] 2024-11-23T15:56:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:56:38,685 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:56:38,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b712f9af2c12,34557,1732377334154 2024-11-23T15:56:38,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14688263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:56:38,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,34557,-1 for getting cluster id 2024-11-23T15:56:38,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:56:38,710 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e52858c2-6096-4906-8ceb-18f6d3f3a43d' 2024-11-23T15:56:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:56:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e52858c2-6096-4906-8ceb-18f6d3f3a43d" 2024-11-23T15:56:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee2f7f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:56:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,34557,-1] 2024-11-23T15:56:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:56:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:56:38,713 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:56:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a9f507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:56:38,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=8026 2024-11-23T15:56:38,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:56:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,34557,1732377334154 2024-11-23T15:56:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62a40fed 2024-11-23T15:56:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:56:38,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:56:38,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:56:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-23T15:56:38,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:56:38,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-23T15:56:38,783 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:38,785 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:56:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:56:38,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741835_1011 (size=428) 2024-11-23T15:56:39,204 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 690fad263b010ff3abe3855319d4fb1a, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:56:39,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741836_1012 (size=79) 2024-11-23T15:56:39,619 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:39,619 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 690fad263b010ff3abe3855319d4fb1a, disabling compactions & flushes 2024-11-23T15:56:39,620 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:39,620 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:39,620 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. after waiting 0 ms 2024-11-23T15:56:39,621 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:39,621 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:39,621 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 690fad263b010ff3abe3855319d4fb1a: Waiting for close lock at 1732377399619Disabling compacts and flushes for region at 1732377399619Disabling writes for close at 1732377399620 (+1 ms)Writing region close event to WAL at 1732377399621 (+1 ms)Closed at 1732377399621 2024-11-23T15:56:39,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:56:39,631 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732377399624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377399624"}]},"ts":"1732377399624"} 2024-11-23T15:56:39,636 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:56:39,638 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:56:39,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377399639"}]},"ts":"1732377399639"} 2024-11-23T15:56:39,646 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-23T15:56:39,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=690fad263b010ff3abe3855319d4fb1a, ASSIGN}] 2024-11-23T15:56:39,652 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=690fad263b010ff3abe3855319d4fb1a, ASSIGN 2024-11-23T15:56:39,654 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=690fad263b010ff3abe3855319d4fb1a, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:56:39,807 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=690fad263b010ff3abe3855319d4fb1a, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:56:39,816 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=690fad263b010ff3abe3855319d4fb1a, ASSIGN because future has completed 2024-11-23T15:56:39,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 690fad263b010ff3abe3855319d4fb1a, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:56:39,984 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:39,985 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 690fad263b010ff3abe3855319d4fb1a, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:56:39,986 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,986 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:39,986 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,986 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,990 INFO [StoreOpener-690fad263b010ff3abe3855319d4fb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,993 INFO [StoreOpener-690fad263b010ff3abe3855319d4fb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690fad263b010ff3abe3855319d4fb1a columnFamilyName testFamily 2024-11-23T15:56:39,993 DEBUG [StoreOpener-690fad263b010ff3abe3855319d4fb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:39,994 INFO [StoreOpener-690fad263b010ff3abe3855319d4fb1a-1 {}] regionserver.HStore(327): Store=690fad263b010ff3abe3855319d4fb1a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:56:39,994 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,995 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,996 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,996 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,996 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:39,999 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:40,002 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:56:40,003 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 690fad263b010ff3abe3855319d4fb1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67794491, jitterRate=0.010216638445854187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:56:40,003 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:56:40,004 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 690fad263b010ff3abe3855319d4fb1a: Running coprocessor pre-open hook at 1732377399987Writing region info on filesystem at 1732377399987Initializing all the Stores at 1732377399990 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377399990Cleaning up temporary data from old regions at 1732377399996 (+6 ms)Running coprocessor post-open hooks at 1732377400003 (+7 ms)Region opened successfully at 1732377400004 (+1 ms) 2024-11-23T15:56:40,006 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., pid=6, masterSystemTime=1732377399974 2024-11-23T15:56:40,009 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:40,009 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:56:40,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=690fad263b010ff3abe3855319d4fb1a, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:40,014 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 690fad263b010ff3abe3855319d4fb1a, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:56:40,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T15:56:40,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 690fad263b010ff3abe3855319d4fb1a, server=b712f9af2c12,41227,1732377334860 in 198 msec 2024-11-23T15:56:40,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T15:56:40,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=690fad263b010ff3abe3855319d4fb1a, ASSIGN in 372 msec 2024-11-23T15:56:40,026 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:56:40,026 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377400026"}]},"ts":"1732377400026"} 2024-11-23T15:56:40,029 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-23T15:56:40,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:56:40,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2590 sec 2024-11-23T15:56:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:56:41,832 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-23T15:56:41,832 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-23T15:56:41,834 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:56:41,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-23T15:56:41,842 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:56:41,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-23T15:56:41,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:56:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,41227,1732377334860, seqNum=-1] 2024-11-23T15:56:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:56:41,850 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:56:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:56:41,870 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:56:41,873 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-23T15:56:41,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-23T15:56:41,883 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-23T15:56:41,885 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-23T15:56:41,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7779330072602738696 2024-11-23T15:56:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377404885, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:56:41,895 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-23T15:56:41,897 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-23T15:56:41,898 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:56:41,900 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:56:41,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7779330072602738698 2024-11-23T15:56:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377404900, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:56:41,905 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:56:41,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=7779330072602738699 2024-11-23T15:56:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377404904, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:56:41,909 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:56:41,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=7779330072602738700 2024-11-23T15:56:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377404909, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:56:41,914 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:56:41,927 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_772733010_22 at /127.0.0.1:57718 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=193 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=8018 (was 8026) 2024-11-23T15:56:41,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=193, ProcessCount=11, AvailableMemoryMB=8016 2024-11-23T15:56:41,942 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:56:41,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-23T15:56:41,947 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:56:41,948 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:41,948 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-23T15:56:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:56:41,950 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:56:41,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741837_1013 (size=414) 2024-11-23T15:56:42,365 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a8f57953b771b9b7bde5739204e9d53, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:56:42,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741838_1014 (size=65) 2024-11-23T15:56:42,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:42,780 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 0a8f57953b771b9b7bde5739204e9d53, disabling compactions & flushes 2024-11-23T15:56:42,780 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:42,780 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:42,780 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. after waiting 0 ms 2024-11-23T15:56:42,780 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:42,781 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:42,781 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a8f57953b771b9b7bde5739204e9d53: Waiting for close lock at 1732377402779Disabling compacts and flushes for region at 1732377402779Disabling writes for close at 1732377402780 (+1 ms)Writing region close event to WAL at 1732377402780Closed at 1732377402780 2024-11-23T15:56:42,784 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:56:42,784 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732377402784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377402784"}]},"ts":"1732377402784"} 2024-11-23T15:56:42,788 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:56:42,789 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:56:42,790 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377402790"}]},"ts":"1732377402790"} 2024-11-23T15:56:42,793 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-23T15:56:42,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a8f57953b771b9b7bde5739204e9d53, ASSIGN}] 2024-11-23T15:56:42,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a8f57953b771b9b7bde5739204e9d53, ASSIGN 2024-11-23T15:56:42,797 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a8f57953b771b9b7bde5739204e9d53, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:56:42,948 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0a8f57953b771b9b7bde5739204e9d53, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:42,953 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a8f57953b771b9b7bde5739204e9d53, ASSIGN because future has completed 2024-11-23T15:56:42,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a8f57953b771b9b7bde5739204e9d53, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:56:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:56:43,120 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:43,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a8f57953b771b9b7bde5739204e9d53, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:56:43,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:43,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,124 INFO [StoreOpener-0a8f57953b771b9b7bde5739204e9d53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,127 INFO [StoreOpener-0a8f57953b771b9b7bde5739204e9d53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a8f57953b771b9b7bde5739204e9d53 columnFamilyName testFamily 2024-11-23T15:56:43,128 DEBUG [StoreOpener-0a8f57953b771b9b7bde5739204e9d53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:43,129 INFO [StoreOpener-0a8f57953b771b9b7bde5739204e9d53-1 {}] regionserver.HStore(327): Store=0a8f57953b771b9b7bde5739204e9d53/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:56:43,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,131 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,131 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,139 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:56:43,187 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 0a8f57953b771b9b7bde5739204e9d53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59695472, jitterRate=-0.11046814918518066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:56:43,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:56:43,190 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0a8f57953b771b9b7bde5739204e9d53: Running coprocessor pre-open hook at 1732377403122Writing region info on filesystem at 1732377403122Initializing all the Stores at 1732377403124 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377403124Cleaning up temporary data from old regions at 1732377403132 (+8 ms)Running coprocessor post-open hooks at 1732377403187 (+55 ms)Region opened successfully at 1732377403189 (+2 ms) 2024-11-23T15:56:43,192 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., pid=9, masterSystemTime=1732377403111 2024-11-23T15:56:43,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:43,199 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:56:43,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0a8f57953b771b9b7bde5739204e9d53, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:43,205 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a8f57953b771b9b7bde5739204e9d53, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:56:43,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T15:56:43,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a8f57953b771b9b7bde5739204e9d53, server=b712f9af2c12,41227,1732377334860 in 254 msec 2024-11-23T15:56:43,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T15:56:43,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a8f57953b771b9b7bde5739204e9d53, ASSIGN in 419 msec 2024-11-23T15:56:43,217 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:56:43,217 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377403217"}]},"ts":"1732377403217"} 2024-11-23T15:56:43,220 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-23T15:56:43,222 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:56:43,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2800 sec 2024-11-23T15:56:43,849 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:56:43,880 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-23T15:56:43,882 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T15:56:43,882 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-23T15:56:44,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T15:56:44,517 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T15:56:44,521 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:56:44,521 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T15:56:44,522 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-23T15:56:44,522 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T15:56:44,525 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-23T15:56:44,525 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-23T15:56:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:56:44,993 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-23T15:56:44,993 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-23T15:56:44,994 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:56:45,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-23T15:56:45,001 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:56:45,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-23T15:56:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:56:45,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:56:45,016 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-23T15:56:45,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T15:56:45,021 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-23T15:56:51,033 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., hostname=b712f9af2c12,41227,1732377334860, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b712f9af2c12:41227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-23T15:56:51,033 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-23T15:56:51,033 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 from cache 2024-11-23T15:56:51,041 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=178 (was 193), ProcessCount=11 (was 11), AvailableMemoryMB=7993 (was 8016) 2024-11-23T15:56:51,048 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=178, ProcessCount=11, AvailableMemoryMB=7993 2024-11-23T15:56:51,051 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:56:51,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-23T15:56:51,055 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:56:51,055 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:51,055 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-23T15:56:51,057 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:56:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-23T15:56:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741839_1015 (size=412) 2024-11-23T15:56:51,471 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d8352eae98dce633f517661da705d92, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:56:51,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741840_1016 (size=63) 2024-11-23T15:56:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-23T15:56:51,885 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:51,885 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6d8352eae98dce633f517661da705d92, disabling compactions & flushes 2024-11-23T15:56:51,885 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:51,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:51,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. after waiting 0 ms 2024-11-23T15:56:51,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:51,886 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:51,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d8352eae98dce633f517661da705d92: Waiting for close lock at 1732377411885Disabling compacts and flushes for region at 1732377411885Disabling writes for close at 1732377411886 (+1 ms)Writing region close event to WAL at 1732377411886Closed at 1732377411886 2024-11-23T15:56:51,889 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:56:51,890 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732377411890"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377411890"}]},"ts":"1732377411890"} 2024-11-23T15:56:51,895 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:56:51,898 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:56:51,898 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377411898"}]},"ts":"1732377411898"} 2024-11-23T15:56:51,901 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-23T15:56:51,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6d8352eae98dce633f517661da705d92, ASSIGN}] 2024-11-23T15:56:51,903 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6d8352eae98dce633f517661da705d92, ASSIGN 2024-11-23T15:56:51,905 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6d8352eae98dce633f517661da705d92, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:56:52,056 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6d8352eae98dce633f517661da705d92, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:52,065 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6d8352eae98dce633f517661da705d92, ASSIGN because future has completed 2024-11-23T15:56:52,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8352eae98dce633f517661da705d92, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:56:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-23T15:56:52,231 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:52,231 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d8352eae98dce633f517661da705d92, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:56:52,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:56:52,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,235 INFO [StoreOpener-6d8352eae98dce633f517661da705d92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,239 INFO [StoreOpener-6d8352eae98dce633f517661da705d92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d8352eae98dce633f517661da705d92 columnFamilyName testFamily 2024-11-23T15:56:52,239 DEBUG [StoreOpener-6d8352eae98dce633f517661da705d92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:56:52,240 INFO [StoreOpener-6d8352eae98dce633f517661da705d92-1 {}] regionserver.HStore(327): Store=6d8352eae98dce633f517661da705d92/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:56:52,240 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,243 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,250 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:56:52,253 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 6d8352eae98dce633f517661da705d92; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68118077, jitterRate=0.015038445591926575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:56:52,254 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d8352eae98dce633f517661da705d92 2024-11-23T15:56:52,254 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6d8352eae98dce633f517661da705d92: Running coprocessor pre-open hook at 1732377412232Writing region info on filesystem at 1732377412232Initializing all the Stores at 1732377412235 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377412235Cleaning up temporary data from old regions at 1732377412245 (+10 ms)Running coprocessor post-open hooks at 1732377412254 (+9 ms)Region opened successfully at 1732377412254 2024-11-23T15:56:52,255 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92., pid=12, masterSystemTime=1732377412220 2024-11-23T15:56:52,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:52,258 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:56:52,259 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6d8352eae98dce633f517661da705d92, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:56:52,262 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8352eae98dce633f517661da705d92, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:56:52,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T15:56:52,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d8352eae98dce633f517661da705d92, server=b712f9af2c12,41227,1732377334860 in 198 msec 2024-11-23T15:56:52,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-23T15:56:52,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6d8352eae98dce633f517661da705d92, ASSIGN in 366 msec 2024-11-23T15:56:52,272 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:56:52,272 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377412272"}]},"ts":"1732377412272"} 2024-11-23T15:56:52,275 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-23T15:56:52,276 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:56:52,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2250 sec 2024-11-23T15:56:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-23T15:56:54,092 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-23T15:56:54,093 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-23T15:56:54,093 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:56:54,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-23T15:56:54,102 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:56:54,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-23T15:56:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:56:54,119 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:56:54,119 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-23T15:56:54,123 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T15:56:54,125 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T15:56:54,127 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T15:56:54,128 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-23T15:56:54,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-23T15:56:54,516 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T15:56:55,885 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-23T15:56:57,770 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:00,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738709' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T15:57:00,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57322 deadline: 1732377426132, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738709' does not exist 2024-11-23T15:57:00,136 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b712f9af2c12,41227,1732377334860 for closing scanner id = 7779330072602738709 for 6d8352eae98dce633f517661da705d92 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738709' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:57:00,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 178), ProcessCount=11 (was 11), AvailableMemoryMB=7998 (was 7993) - AvailableMemoryMB LEAK? - 2024-11-23T15:57:00,147 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=7997 2024-11-23T15:57:00,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-23T15:57:00,153 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:00,153 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:00,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-23T15:57:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:57:00,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:00,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741841_1017 (size=423) 2024-11-23T15:57:00,571 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2759533b2a727b251d39de591f0c199, NAME => 'testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:00,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741842_1018 (size=74) 2024-11-23T15:57:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-23T15:57:00,987 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:00,987 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing d2759533b2a727b251d39de591f0c199, disabling compactions & flushes 2024-11-23T15:57:00,988 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:00,988 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:00,988 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. after waiting 0 ms 2024-11-23T15:57:00,988 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:00,988 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:00,989 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2759533b2a727b251d39de591f0c199: Waiting for close lock at 1732377420987Disabling compacts and flushes for region at 1732377420987Disabling writes for close at 1732377420988 (+1 ms)Writing region close event to WAL at 1732377420988Closed at 1732377420988 2024-11-23T15:57:00,992 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:00,992 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732377420992"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377420992"}]},"ts":"1732377420992"} 2024-11-23T15:57:00,997 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:00,999 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:01,000 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377420999"}]},"ts":"1732377420999"} 2024-11-23T15:57:01,003 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-23T15:57:01,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2759533b2a727b251d39de591f0c199, ASSIGN}] 2024-11-23T15:57:01,006 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2759533b2a727b251d39de591f0c199, ASSIGN 2024-11-23T15:57:01,007 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2759533b2a727b251d39de591f0c199, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:01,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d2759533b2a727b251d39de591f0c199, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:57:01,167 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2759533b2a727b251d39de591f0c199, ASSIGN because future has completed 2024-11-23T15:57:01,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2759533b2a727b251d39de591f0c199, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:01,330 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:01,330 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => d2759533b2a727b251d39de591f0c199, NAME => 'testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:01,331 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,331 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:01,331 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,331 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,333 INFO [StoreOpener-d2759533b2a727b251d39de591f0c199-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,337 INFO [StoreOpener-d2759533b2a727b251d39de591f0c199-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2759533b2a727b251d39de591f0c199 columnFamilyName testFamily 2024-11-23T15:57:01,337 DEBUG [StoreOpener-d2759533b2a727b251d39de591f0c199-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:01,338 INFO [StoreOpener-d2759533b2a727b251d39de591f0c199-1 {}] regionserver.HStore(327): Store=d2759533b2a727b251d39de591f0c199/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:01,338 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,340 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,340 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,341 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,341 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,344 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,347 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:01,348 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened d2759533b2a727b251d39de591f0c199; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68930848, jitterRate=0.027149677276611328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:01,348 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:01,349 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for d2759533b2a727b251d39de591f0c199: Running coprocessor pre-open hook at 1732377421331Writing region info on filesystem at 1732377421331Initializing all the Stores at 1732377421333 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377421333Cleaning up temporary data from old regions at 1732377421341 (+8 ms)Running coprocessor post-open hooks at 1732377421348 (+7 ms)Region opened successfully at 1732377421349 (+1 ms) 2024-11-23T15:57:01,350 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199., pid=15, masterSystemTime=1732377421323 2024-11-23T15:57:01,353 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:01,353 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:01,354 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d2759533b2a727b251d39de591f0c199, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:01,357 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2759533b2a727b251d39de591f0c199, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:01,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-23T15:57:01,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure d2759533b2a727b251d39de591f0c199, server=b712f9af2c12,41227,1732377334860 in 191 msec 2024-11-23T15:57:01,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T15:57:01,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2759533b2a727b251d39de591f0c199, ASSIGN in 358 msec 2024-11-23T15:57:01,366 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:01,366 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377421366"}]},"ts":"1732377421366"} 2024-11-23T15:57:01,369 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-23T15:57:01,370 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:01,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2210 sec 2024-11-23T15:57:01,889 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-23T15:57:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:57:03,192 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-23T15:57:03,192 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-23T15:57:03,193 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:03,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-23T15:57:03,203 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:03,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-23T15:57:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:03,216 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:03,217 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-23T15:57:03,219 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:57:03,219 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-23T15:57:03,222 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-23T15:57:03,223 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-23T15:57:03,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7779330072602738713 2024-11-23T15:57:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377426222, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:57:03,227 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-23T15:57:03,229 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-23T15:57:03,230 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:57:03,231 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:57:03,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7779330072602738715 2024-11-23T15:57:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377426231, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:57:03,235 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:57:03,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=7779330072602738716 2024-11-23T15:57:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377426235, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:57:03,239 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:57:03,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=7779330072602738717 2024-11-23T15:57:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57322 deadline: 1732377426239, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T15:57:03,244 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T15:57:03,252 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_772733010_22 at /127.0.0.1:39302 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=7988 (was 7997) 2024-11-23T15:57:03,260 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=7988 2024-11-23T15:57:03,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-23T15:57:03,270 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:03,270 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:03,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-23T15:57:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-23T15:57:03,271 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:03,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741843_1019 (size=418) 2024-11-23T15:57:03,687 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46ee52ab2a0dcd7d79aec6f76a473fd3, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:03,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741844_1020 (size=69) 2024-11-23T15:57:04,102 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:04,102 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 46ee52ab2a0dcd7d79aec6f76a473fd3, disabling compactions & flushes 2024-11-23T15:57:04,103 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,103 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,103 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. after waiting 0 ms 2024-11-23T15:57:04,103 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,103 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,104 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46ee52ab2a0dcd7d79aec6f76a473fd3: Waiting for close lock at 1732377424102Disabling compacts and flushes for region at 1732377424102Disabling writes for close at 1732377424103 (+1 ms)Writing region close event to WAL at 1732377424103Closed at 1732377424103 2024-11-23T15:57:04,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:04,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732377424108"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377424108"}]},"ts":"1732377424108"} 2024-11-23T15:57:04,114 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:04,116 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:04,117 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377424116"}]},"ts":"1732377424116"} 2024-11-23T15:57:04,120 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-23T15:57:04,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=46ee52ab2a0dcd7d79aec6f76a473fd3, ASSIGN}] 2024-11-23T15:57:04,121 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=46ee52ab2a0dcd7d79aec6f76a473fd3, ASSIGN 2024-11-23T15:57:04,123 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=46ee52ab2a0dcd7d79aec6f76a473fd3, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:04,274 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=46ee52ab2a0dcd7d79aec6f76a473fd3, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-23T15:57:04,280 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=46ee52ab2a0dcd7d79aec6f76a473fd3, ASSIGN because future has completed 2024-11-23T15:57:04,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46ee52ab2a0dcd7d79aec6f76a473fd3, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:04,443 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,444 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 46ee52ab2a0dcd7d79aec6f76a473fd3, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:04,444 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,444 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:04,445 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,445 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,447 INFO [StoreOpener-46ee52ab2a0dcd7d79aec6f76a473fd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,449 INFO [StoreOpener-46ee52ab2a0dcd7d79aec6f76a473fd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46ee52ab2a0dcd7d79aec6f76a473fd3 columnFamilyName testFamily 2024-11-23T15:57:04,449 DEBUG [StoreOpener-46ee52ab2a0dcd7d79aec6f76a473fd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:04,450 INFO [StoreOpener-46ee52ab2a0dcd7d79aec6f76a473fd3-1 {}] regionserver.HStore(327): Store=46ee52ab2a0dcd7d79aec6f76a473fd3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:04,450 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,451 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,452 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,453 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,453 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,455 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,458 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:04,458 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 46ee52ab2a0dcd7d79aec6f76a473fd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75358538, jitterRate=0.12292972207069397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:04,459 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:04,459 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 46ee52ab2a0dcd7d79aec6f76a473fd3: Running coprocessor pre-open hook at 1732377424445Writing region info on filesystem at 1732377424445Initializing all the Stores at 1732377424447 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377424447Cleaning up temporary data from old regions at 1732377424453 (+6 ms)Running coprocessor post-open hooks at 1732377424459 (+6 ms)Region opened successfully at 1732377424459 2024-11-23T15:57:04,460 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., pid=18, masterSystemTime=1732377424435 2024-11-23T15:57:04,463 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,463 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:04,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=46ee52ab2a0dcd7d79aec6f76a473fd3, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:04,467 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46ee52ab2a0dcd7d79aec6f76a473fd3, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:04,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-23T15:57:04,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 46ee52ab2a0dcd7d79aec6f76a473fd3, server=b712f9af2c12,41227,1732377334860 in 188 msec 2024-11-23T15:57:04,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-23T15:57:04,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=46ee52ab2a0dcd7d79aec6f76a473fd3, ASSIGN in 352 msec 2024-11-23T15:57:04,477 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:04,477 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377424477"}]},"ts":"1732377424477"} 2024-11-23T15:57:04,479 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-23T15:57:04,480 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:04,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2140 sec 2024-11-23T15:57:04,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-23T15:57:04,516 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:04,517 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-23T15:57:04,517 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-23T15:57:06,292 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-23T15:57:06,292 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-23T15:57:06,293 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:06,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-23T15:57:06,303 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:06,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-23T15:57:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:06,326 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:06,326 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-23T15:57:06,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T15:57:06,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T15:57:06,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:07,340 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b712f9af2c12:41227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-23T15:57:07,341 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-23T15:57:07,341 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 from cache 2024-11-23T15:57:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-23T15:57:07,892 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-23T15:57:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:08,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T15:57:09,340 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b712f9af2c12:41227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=984ms, rpcTimeout=976ms 2024-11-23T15:57:09,340 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=984ms, rpcTimeout=976ms 2024-11-23T15:57:09,341 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 from cache 2024-11-23T15:57:09,359 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=7980 (was 7988) 2024-11-23T15:57:09,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=7979 2024-11-23T15:57:09,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-23T15:57:09,377 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:09,377 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:09,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-23T15:57:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:57:09,379 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:09,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741845_1021 (size=416) 2024-11-23T15:57:09,796 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a034b7b0fb6a24a4005d31cccb2f9271, NAME => 'testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:09,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741846_1022 (size=67) 2024-11-23T15:57:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-23T15:57:10,210 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:10,211 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing a034b7b0fb6a24a4005d31cccb2f9271, disabling compactions & flushes 2024-11-23T15:57:10,211 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,211 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,211 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. after waiting 0 ms 2024-11-23T15:57:10,212 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,212 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a034b7b0fb6a24a4005d31cccb2f9271: Waiting for close lock at 1732377430211Disabling compacts and flushes for region at 1732377430211Disabling writes for close at 1732377430211Writing region close event to WAL at 1732377430212 (+1 ms)Closed at 1732377430212 2024-11-23T15:57:10,219 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:10,220 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732377430219"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377430219"}]},"ts":"1732377430219"} 2024-11-23T15:57:10,224 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:10,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:10,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377430226"}]},"ts":"1732377430226"} 2024-11-23T15:57:10,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-23T15:57:10,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a034b7b0fb6a24a4005d31cccb2f9271, ASSIGN}] 2024-11-23T15:57:10,231 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a034b7b0fb6a24a4005d31cccb2f9271, ASSIGN 2024-11-23T15:57:10,233 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a034b7b0fb6a24a4005d31cccb2f9271, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:10,384 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=a034b7b0fb6a24a4005d31cccb2f9271, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:10,393 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a034b7b0fb6a24a4005d31cccb2f9271, ASSIGN because future has completed 2024-11-23T15:57:10,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure a034b7b0fb6a24a4005d31cccb2f9271, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:57:10,562 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,562 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => a034b7b0fb6a24a4005d31cccb2f9271, NAME => 'testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:10,563 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,563 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:10,563 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,563 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,565 INFO [StoreOpener-a034b7b0fb6a24a4005d31cccb2f9271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,567 INFO [StoreOpener-a034b7b0fb6a24a4005d31cccb2f9271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a034b7b0fb6a24a4005d31cccb2f9271 columnFamilyName testFamily 2024-11-23T15:57:10,567 DEBUG [StoreOpener-a034b7b0fb6a24a4005d31cccb2f9271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:10,568 INFO [StoreOpener-a034b7b0fb6a24a4005d31cccb2f9271-1 {}] regionserver.HStore(327): Store=a034b7b0fb6a24a4005d31cccb2f9271/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:10,568 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,569 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,569 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,570 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,570 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,572 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,576 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:10,577 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened a034b7b0fb6a24a4005d31cccb2f9271; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68646370, jitterRate=0.022910624742507935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:10,577 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:10,577 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for a034b7b0fb6a24a4005d31cccb2f9271: Running coprocessor pre-open hook at 1732377430563Writing region info on filesystem at 1732377430563Initializing all the Stores at 1732377430564 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377430564Cleaning up temporary data from old regions at 1732377430570 (+6 ms)Running coprocessor post-open hooks at 1732377430577 (+7 ms)Region opened successfully at 1732377430577 2024-11-23T15:57:10,579 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271., pid=21, masterSystemTime=1732377430549 2024-11-23T15:57:10,582 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,582 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:10,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=a034b7b0fb6a24a4005d31cccb2f9271, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:10,588 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure a034b7b0fb6a24a4005d31cccb2f9271, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:10,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-23T15:57:10,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure a034b7b0fb6a24a4005d31cccb2f9271, server=b712f9af2c12,41227,1732377334860 in 196 msec 2024-11-23T15:57:10,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-23T15:57:10,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a034b7b0fb6a24a4005d31cccb2f9271, ASSIGN in 363 msec 2024-11-23T15:57:10,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:10,597 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377430597"}]},"ts":"1732377430597"} 2024-11-23T15:57:10,599 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-23T15:57:10,601 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:10,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2280 sec 2024-11-23T15:57:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:57:12,410 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-23T15:57:12,410 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-23T15:57:12,411 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:12,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-23T15:57:12,419 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:12,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-23T15:57:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:12,433 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:12,433 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-23T15:57:12,436 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T15:57:12,438 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T15:57:12,439 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T15:57:12,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-23T15:57:13,894 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-23T15:57:14,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-23T15:57:14,516 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:15,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738728' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T15:57:15,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57322 deadline: 1732377436446, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738728' does not exist 2024-11-23T15:57:15,450 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b712f9af2c12,41227,1732377334860 for closing scanner id = 7779330072602738728 for a034b7b0fb6a24a4005d31cccb2f9271 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738728' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:57:15,456 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=117 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=7973 (was 7979) 2024-11-23T15:57:15,465 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=117, ProcessCount=11, AvailableMemoryMB=7973 2024-11-23T15:57:15,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-23T15:57:15,471 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:15,472 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:15,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-23T15:57:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-23T15:57:15,473 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:15,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741847_1023 (size=411) 2024-11-23T15:57:15,888 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aee6f85aed61263650d05932716e6455, NAME => 'testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741848_1024 (size=62) 2024-11-23T15:57:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-23T15:57:16,067 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:16,303 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:16,303 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing aee6f85aed61263650d05932716e6455, disabling compactions & flushes 2024-11-23T15:57:16,304 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. after waiting 0 ms 2024-11-23T15:57:16,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,304 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,305 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for aee6f85aed61263650d05932716e6455: Waiting for close lock at 1732377436303Disabling compacts and flushes for region at 1732377436303Disabling writes for close at 1732377436304 (+1 ms)Writing region close event to WAL at 1732377436304Closed at 1732377436304 2024-11-23T15:57:16,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:16,309 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732377436309"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377436309"}]},"ts":"1732377436309"} 2024-11-23T15:57:16,313 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:16,315 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:16,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377436315"}]},"ts":"1732377436315"} 2024-11-23T15:57:16,318 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-23T15:57:16,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=aee6f85aed61263650d05932716e6455, ASSIGN}] 2024-11-23T15:57:16,321 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=aee6f85aed61263650d05932716e6455, ASSIGN 2024-11-23T15:57:16,322 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=aee6f85aed61263650d05932716e6455, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:16,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=aee6f85aed61263650d05932716e6455, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:16,477 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=aee6f85aed61263650d05932716e6455, ASSIGN because future has completed 2024-11-23T15:57:16,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure aee6f85aed61263650d05932716e6455, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-23T15:57:16,642 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,643 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => aee6f85aed61263650d05932716e6455, NAME => 'testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:16,643 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,643 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:16,643 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,643 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,646 INFO [StoreOpener-aee6f85aed61263650d05932716e6455-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,649 INFO [StoreOpener-aee6f85aed61263650d05932716e6455-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aee6f85aed61263650d05932716e6455 columnFamilyName testFamily 2024-11-23T15:57:16,649 DEBUG [StoreOpener-aee6f85aed61263650d05932716e6455-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:16,650 INFO [StoreOpener-aee6f85aed61263650d05932716e6455-1 {}] regionserver.HStore(327): Store=aee6f85aed61263650d05932716e6455/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:16,650 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,652 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,653 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,653 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,654 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,657 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,661 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:16,662 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened aee6f85aed61263650d05932716e6455; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71181458, jitterRate=0.0606863796710968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:16,662 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:16,663 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for aee6f85aed61263650d05932716e6455: Running coprocessor pre-open hook at 1732377436644Writing region info on filesystem at 1732377436644Initializing all the Stores at 1732377436646 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377436646Cleaning up temporary data from old regions at 1732377436654 (+8 ms)Running coprocessor post-open hooks at 1732377436662 (+8 ms)Region opened successfully at 1732377436663 (+1 ms) 2024-11-23T15:57:16,664 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455., pid=24, masterSystemTime=1732377436632 2024-11-23T15:57:16,667 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,668 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:16,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=aee6f85aed61263650d05932716e6455, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:16,672 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure aee6f85aed61263650d05932716e6455, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:16,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-23T15:57:16,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure aee6f85aed61263650d05932716e6455, server=b712f9af2c12,41227,1732377334860 in 196 msec 2024-11-23T15:57:16,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-23T15:57:16,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=aee6f85aed61263650d05932716e6455, ASSIGN in 357 msec 2024-11-23T15:57:16,681 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:16,682 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377436682"}]},"ts":"1732377436682"} 2024-11-23T15:57:16,685 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-23T15:57:16,686 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:16,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2190 sec 2024-11-23T15:57:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-23T15:57:18,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-23T15:57:18,493 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-23T15:57:18,493 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:18,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-23T15:57:18,506 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:18,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-23T15:57:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:18,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:18,523 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-23T15:57:18,528 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T15:57:18,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T15:57:18,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T15:57:18,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-23T15:57:19,563 INFO [master/b712f9af2c12:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T15:57:19,564 INFO [master/b712f9af2c12:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T15:57:19,897 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-23T15:57:21,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738732' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T15:57:21,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57322 deadline: 1732377442547, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738732' does not exist 2024-11-23T15:57:21,550 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b712f9af2c12,41227,1732377334860 for closing scanner id = 7779330072602738732 for aee6f85aed61263650d05932716e6455 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738732' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:57:21,556 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=108 (was 117), ProcessCount=11 (was 11), AvailableMemoryMB=7969 (was 7973) 2024-11-23T15:57:21,565 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=108, ProcessCount=11, AvailableMemoryMB=7969 2024-11-23T15:57:21,568 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:21,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-23T15:57:21,571 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:21,572 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:21,572 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-23T15:57:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:57:21,573 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:21,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741849_1025 (size=419) 2024-11-23T15:57:21,990 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e383d1d2091ae721c8fde2fab32043d5, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:22,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741850_1026 (size=70) 2024-11-23T15:57:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-23T15:57:22,147 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:22,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:22,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing e383d1d2091ae721c8fde2fab32043d5, disabling compactions & flushes 2024-11-23T15:57:22,404 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,405 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,405 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. after waiting 0 ms 2024-11-23T15:57:22,405 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,405 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,405 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e383d1d2091ae721c8fde2fab32043d5: Waiting for close lock at 1732377442404Disabling compacts and flushes for region at 1732377442404Disabling writes for close at 1732377442405 (+1 ms)Writing region close event to WAL at 1732377442405Closed at 1732377442405 2024-11-23T15:57:22,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:22,409 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732377442408"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377442408"}]},"ts":"1732377442408"} 2024-11-23T15:57:22,412 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:22,413 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:22,414 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377442414"}]},"ts":"1732377442414"} 2024-11-23T15:57:22,416 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-23T15:57:22,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e383d1d2091ae721c8fde2fab32043d5, ASSIGN}] 2024-11-23T15:57:22,418 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e383d1d2091ae721c8fde2fab32043d5, ASSIGN 2024-11-23T15:57:22,419 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e383d1d2091ae721c8fde2fab32043d5, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:22,571 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=e383d1d2091ae721c8fde2fab32043d5, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:22,579 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e383d1d2091ae721c8fde2fab32043d5, ASSIGN because future has completed 2024-11-23T15:57:22,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure e383d1d2091ae721c8fde2fab32043d5, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:57:22,746 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => e383d1d2091ae721c8fde2fab32043d5, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:22,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:22,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,749 INFO [StoreOpener-e383d1d2091ae721c8fde2fab32043d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,751 INFO [StoreOpener-e383d1d2091ae721c8fde2fab32043d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e383d1d2091ae721c8fde2fab32043d5 columnFamilyName testFamily 2024-11-23T15:57:22,751 DEBUG [StoreOpener-e383d1d2091ae721c8fde2fab32043d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:22,751 INFO [StoreOpener-e383d1d2091ae721c8fde2fab32043d5-1 {}] regionserver.HStore(327): Store=e383d1d2091ae721c8fde2fab32043d5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:22,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,755 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:22,759 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened e383d1d2091ae721c8fde2fab32043d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60044542, jitterRate=-0.10526660084724426}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:22,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:22,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for e383d1d2091ae721c8fde2fab32043d5: Running coprocessor pre-open hook at 1732377442747Writing region info on filesystem at 1732377442747Initializing all the Stores at 1732377442749 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377442749Cleaning up temporary data from old regions at 1732377442753 (+4 ms)Running coprocessor post-open hooks at 1732377442759 (+6 ms)Region opened successfully at 1732377442759 2024-11-23T15:57:22,760 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., pid=27, masterSystemTime=1732377442735 2024-11-23T15:57:22,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,763 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:22,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=e383d1d2091ae721c8fde2fab32043d5, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:22,767 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure e383d1d2091ae721c8fde2fab32043d5, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:22,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-23T15:57:22,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure e383d1d2091ae721c8fde2fab32043d5, server=b712f9af2c12,41227,1732377334860 in 188 msec 2024-11-23T15:57:22,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-23T15:57:22,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e383d1d2091ae721c8fde2fab32043d5, ASSIGN in 353 msec 2024-11-23T15:57:22,774 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:22,774 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377442774"}]},"ts":"1732377442774"} 2024-11-23T15:57:22,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-23T15:57:22,778 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:22,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2100 sec 2024-11-23T15:57:24,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-23T15:57:24,517 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:24,519 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-23T15:57:24,519 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:57:24,610 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-23T15:57:24,610 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-23T15:57:24,610 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:24,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-23T15:57:24,618 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:24,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-23T15:57:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:24,634 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:24,634 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-23T15:57:24,637 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T15:57:24,639 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-23T15:57:25,900 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-23T15:57:28,253 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:30,639 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., hostname=b712f9af2c12,41227,1732377334860, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b712f9af2c12:41227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-11-23T15:57:30,640 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-11-23T15:57:30,640 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 from cache 2024-11-23T15:57:30,654 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=91 (was 108), ProcessCount=11 (was 11), AvailableMemoryMB=7957 (was 7969) 2024-11-23T15:57:30,662 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=91, ProcessCount=11, AvailableMemoryMB=7956 2024-11-23T15:57:30,664 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:30,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-23T15:57:30,667 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:30,668 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:30,668 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-23T15:57:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-23T15:57:30,669 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:30,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741851_1027 (size=407) 2024-11-23T15:57:31,086 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 018e567d025663812ad1302286501c06, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:31,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741852_1028 (size=58) 2024-11-23T15:57:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-23T15:57:31,502 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:31,502 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 018e567d025663812ad1302286501c06, disabling compactions & flushes 2024-11-23T15:57:31,502 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,502 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,503 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. after waiting 0 ms 2024-11-23T15:57:31,503 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,503 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,503 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 018e567d025663812ad1302286501c06: Waiting for close lock at 1732377451502Disabling compacts and flushes for region at 1732377451502Disabling writes for close at 1732377451503 (+1 ms)Writing region close event to WAL at 1732377451503Closed at 1732377451503 2024-11-23T15:57:31,507 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:31,507 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732377451507"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377451507"}]},"ts":"1732377451507"} 2024-11-23T15:57:31,512 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:31,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:31,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377451514"}]},"ts":"1732377451514"} 2024-11-23T15:57:31,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-23T15:57:31,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=018e567d025663812ad1302286501c06, ASSIGN}] 2024-11-23T15:57:31,518 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=018e567d025663812ad1302286501c06, ASSIGN 2024-11-23T15:57:31,519 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=018e567d025663812ad1302286501c06, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:31,671 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=018e567d025663812ad1302286501c06, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:31,679 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=018e567d025663812ad1302286501c06, ASSIGN because future has completed 2024-11-23T15:57:31,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 018e567d025663812ad1302286501c06, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-23T15:57:31,841 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 018e567d025663812ad1302286501c06, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:31,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:31,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,844 INFO [StoreOpener-018e567d025663812ad1302286501c06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,846 INFO [StoreOpener-018e567d025663812ad1302286501c06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 018e567d025663812ad1302286501c06 columnFamilyName testFamily 2024-11-23T15:57:31,846 DEBUG [StoreOpener-018e567d025663812ad1302286501c06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:31,846 INFO [StoreOpener-018e567d025663812ad1302286501c06-1 {}] regionserver.HStore(327): Store=018e567d025663812ad1302286501c06/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:31,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06 2024-11-23T15:57:31,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06 2024-11-23T15:57:31,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:31,854 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 018e567d025663812ad1302286501c06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72302779, jitterRate=0.07739536464214325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:31,854 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 018e567d025663812ad1302286501c06 2024-11-23T15:57:31,855 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 018e567d025663812ad1302286501c06: Running coprocessor pre-open hook at 1732377451842Writing region info on filesystem at 1732377451842Initializing all the Stores at 1732377451843 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377451843Cleaning up temporary data from old regions at 1732377451849 (+6 ms)Running coprocessor post-open hooks at 1732377451854 (+5 ms)Region opened successfully at 1732377451855 (+1 ms) 2024-11-23T15:57:31,856 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06., pid=30, masterSystemTime=1732377451833 2024-11-23T15:57:31,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,859 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:31,860 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=018e567d025663812ad1302286501c06, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:31,863 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 018e567d025663812ad1302286501c06, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:31,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-23T15:57:31,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 018e567d025663812ad1302286501c06, server=b712f9af2c12,41227,1732377334860 in 185 msec 2024-11-23T15:57:31,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-23T15:57:31,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=018e567d025663812ad1302286501c06, ASSIGN in 350 msec 2024-11-23T15:57:31,871 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:31,871 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377451871"}]},"ts":"1732377451871"} 2024-11-23T15:57:31,874 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-23T15:57:31,875 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:31,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2110 sec 2024-11-23T15:57:31,903 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-23T15:57:33,219 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:57:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-23T15:57:33,691 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-23T15:57:33,692 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-23T15:57:33,693 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:33,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-23T15:57:33,702 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:33,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-23T15:57:33,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:33,714 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:33,714 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-23T15:57:33,717 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T15:57:33,720 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T15:57:33,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T15:57:33,722 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-23T15:57:34,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-23T15:57:34,517 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:37,347 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:38,547 DEBUG [master/b712f9af2c12:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T15:57:39,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738740' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T15:57:39,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57322 deadline: 1732377465737, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738740' does not exist 2024-11-23T15:57:39,741 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b712f9af2c12,41227,1732377334860 for closing scanner id = 7779330072602738740 for 018e567d025663812ad1302286501c06 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7779330072602738740' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:57:39,746 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=99 (was 91) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7938 (was 7956) 2024-11-23T15:57:39,753 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=99, ProcessCount=11, AvailableMemoryMB=7938 2024-11-23T15:57:39,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:57:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-23T15:57:39,759 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:57:39,759 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:39,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-23T15:57:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:57:39,761 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:57:39,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741853_1029 (size=423) 2024-11-23T15:57:40,177 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fadf4a079dfa037ada244ade41166f38, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61 2024-11-23T15:57:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741854_1030 (size=74) 2024-11-23T15:57:40,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-23T15:57:40,590 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:40,591 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing fadf4a079dfa037ada244ade41166f38, disabling compactions & flushes 2024-11-23T15:57:40,591 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,591 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,591 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. after waiting 0 ms 2024-11-23T15:57:40,591 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,591 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,592 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for fadf4a079dfa037ada244ade41166f38: Waiting for close lock at 1732377460590Disabling compacts and flushes for region at 1732377460590Disabling writes for close at 1732377460591 (+1 ms)Writing region close event to WAL at 1732377460591Closed at 1732377460591 2024-11-23T15:57:40,594 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:57:40,595 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732377460594"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732377460594"}]},"ts":"1732377460594"} 2024-11-23T15:57:40,598 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:57:40,600 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:57:40,601 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377460600"}]},"ts":"1732377460600"} 2024-11-23T15:57:40,604 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-23T15:57:40,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fadf4a079dfa037ada244ade41166f38, ASSIGN}] 2024-11-23T15:57:40,606 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fadf4a079dfa037ada244ade41166f38, ASSIGN 2024-11-23T15:57:40,607 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fadf4a079dfa037ada244ade41166f38, ASSIGN; state=OFFLINE, location=b712f9af2c12,41227,1732377334860; forceNewPlan=false, retain=false 2024-11-23T15:57:40,758 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=fadf4a079dfa037ada244ade41166f38, regionState=OPENING, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:40,765 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fadf4a079dfa037ada244ade41166f38, ASSIGN because future has completed 2024-11-23T15:57:40,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure fadf4a079dfa037ada244ade41166f38, server=b712f9af2c12,41227,1732377334860}] 2024-11-23T15:57:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:57:40,932 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,932 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => fadf4a079dfa037ada244ade41166f38, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:57:40,933 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,933 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:57:40,933 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,933 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,935 INFO [StoreOpener-fadf4a079dfa037ada244ade41166f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,938 INFO [StoreOpener-fadf4a079dfa037ada244ade41166f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fadf4a079dfa037ada244ade41166f38 columnFamilyName testFamily 2024-11-23T15:57:40,938 DEBUG [StoreOpener-fadf4a079dfa037ada244ade41166f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:57:40,939 INFO [StoreOpener-fadf4a079dfa037ada244ade41166f38-1 {}] regionserver.HStore(327): Store=fadf4a079dfa037ada244ade41166f38/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:57:40,939 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,940 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,941 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,942 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,942 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,945 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,950 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:57:40,951 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened fadf4a079dfa037ada244ade41166f38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73355896, jitterRate=0.09308803081512451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:57:40,951 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:40,951 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for fadf4a079dfa037ada244ade41166f38: Running coprocessor pre-open hook at 1732377460933Writing region info on filesystem at 1732377460933Initializing all the Stores at 1732377460935 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732377460935Cleaning up temporary data from old regions at 1732377460942 (+7 ms)Running coprocessor post-open hooks at 1732377460951 (+9 ms)Region opened successfully at 1732377460951 2024-11-23T15:57:40,952 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., pid=33, masterSystemTime=1732377460920 2024-11-23T15:57:40,955 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,955 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:40,956 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=fadf4a079dfa037ada244ade41166f38, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,41227,1732377334860 2024-11-23T15:57:40,958 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure fadf4a079dfa037ada244ade41166f38, server=b712f9af2c12,41227,1732377334860 because future has completed 2024-11-23T15:57:40,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-23T15:57:40,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure fadf4a079dfa037ada244ade41166f38, server=b712f9af2c12,41227,1732377334860 in 194 msec 2024-11-23T15:57:40,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-23T15:57:40,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fadf4a079dfa037ada244ade41166f38, ASSIGN in 357 msec 2024-11-23T15:57:40,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:57:40,965 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732377460965"}]},"ts":"1732377460965"} 2024-11-23T15:57:40,967 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-23T15:57:40,968 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:57:40,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2120 sec 2024-11-23T15:57:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:57:42,792 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-23T15:57:42,792 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-23T15:57:42,793 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:42,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-23T15:57:42,803 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:57:42,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-23T15:57:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:42,815 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T15:57:42,815 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-23T15:57:42,818 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T15:57:42,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T15:57:43,820 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b712f9af2c12:41227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-23T15:57:43,821 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-23T15:57:43,821 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 from cache 2024-11-23T15:57:43,908 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-23T15:57:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-23T15:57:44,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-23T15:57:44,516 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T15:57:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2] 2024-11-23T15:57:44,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T15:57:45,820 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b712f9af2c12:41227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=981ms 2024-11-23T15:57:45,821 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=981ms 2024-11-23T15:57:45,821 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., hostname=b712f9af2c12,41227,1732377334860, seqNum=2 from cache 2024-11-23T15:57:45,835 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=91 (was 99), ProcessCount=11 (was 11), AvailableMemoryMB=7941 (was 7938) - AvailableMemoryMB LEAK? - 2024-11-23T15:57:45,835 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:57:45,836 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:57:45,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:45,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:45,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:57:45,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T15:57:45,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:57:45,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:57:45,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:45,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:45,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:57:45,843 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T15:57:45,843 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=385927850, stopped=false 2024-11-23T15:57:45,843 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b712f9af2c12,34557,1732377334154 2024-11-23T15:57:45,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:57:45,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:57:45,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:57:45,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:57:45,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T15:57:45,910 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:57:45,910 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:57:45,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:57:45,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:45,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:57:45,912 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b712f9af2c12,41227,1732377334860' ***** 2024-11-23T15:57:45,912 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T15:57:45,912 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T15:57:45,913 INFO [RS:0;b712f9af2c12:41227 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T15:57:45,913 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T15:57:45,913 INFO [RS:0;b712f9af2c12:41227 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T15:57:45,914 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for 46ee52ab2a0dcd7d79aec6f76a473fd3 2024-11-23T15:57:45,915 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for e383d1d2091ae721c8fde2fab32043d5 2024-11-23T15:57:45,915 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for a034b7b0fb6a24a4005d31cccb2f9271 2024-11-23T15:57:45,915 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for d2759533b2a727b251d39de591f0c199 2024-11-23T15:57:45,915 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d8352eae98dce633f517661da705d92 2024-11-23T15:57:45,915 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a8f57953b771b9b7bde5739204e9d53 2024-11-23T15:57:45,916 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for aee6f85aed61263650d05932716e6455 2024-11-23T15:57:45,916 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:45,916 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46ee52ab2a0dcd7d79aec6f76a473fd3, disabling compactions & flushes 2024-11-23T15:57:45,916 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:57:45,916 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:45,916 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for 018e567d025663812ad1302286501c06 2024-11-23T15:57:45,916 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:45,916 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(959): stopping server b712f9af2c12,41227,1732377334860 2024-11-23T15:57:45,916 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. after waiting 0 ms 2024-11-23T15:57:45,916 INFO [RS:0;b712f9af2c12:41227 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:57:45,916 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:45,917 INFO [RS:0;b712f9af2c12:41227 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b712f9af2c12:41227. 2024-11-23T15:57:45,917 DEBUG [RS:0;b712f9af2c12:41227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:57:45,917 DEBUG [RS:0;b712f9af2c12:41227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:45,917 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T15:57:45,917 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T15:57:45,918 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T15:57:45,918 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T15:57:45,918 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-23T15:57:45,918 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:57:45,918 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:57:45,918 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:57:45,919 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:57:45,919 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 46ee52ab2a0dcd7d79aec6f76a473fd3, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, e383d1d2091ae721c8fde2fab32043d5, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:45,919 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:57:45,921 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-23T15:57:45,921 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 46ee52ab2a0dcd7d79aec6f76a473fd3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:45,974 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/.tmp/testFamily/713e6ab5f50f476abf97904ff52bfc8a is 61, key is row-0/testFamily:testQualifier/1732377426311/Put/seqid=0 2024-11-23T15:57:45,980 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/info/7815248bf11847ec9b3edbadaa9f1050 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a./info:regioninfo/1732377400010/Put/seqid=0 2024-11-23T15:57:45,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741855_1031 (size=5049) 2024-11-23T15:57:45,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741856_1032 (size=20564) 2024-11-23T15:57:46,119 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 46ee52ab2a0dcd7d79aec6f76a473fd3, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, e383d1d2091ae721c8fde2fab32043d5, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:46,319 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 46ee52ab2a0dcd7d79aec6f76a473fd3, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, e383d1d2091ae721c8fde2fab32043d5, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:46,341 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-23T15:57:46,385 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/.tmp/testFamily/713e6ab5f50f476abf97904ff52bfc8a 2024-11-23T15:57:46,389 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/info/7815248bf11847ec9b3edbadaa9f1050 2024-11-23T15:57:46,432 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/.tmp/testFamily/713e6ab5f50f476abf97904ff52bfc8a as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/testFamily/713e6ab5f50f476abf97904ff52bfc8a 2024-11-23T15:57:46,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:46,441 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/testFamily/713e6ab5f50f476abf97904ff52bfc8a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:46,449 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46ee52ab2a0dcd7d79aec6f76a473fd3 in 528ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:46,451 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/ns/ad7d8c8bebf84967949a244772065cff is 43, key is default/ns:d/1732377398492/Put/seqid=0 2024-11-23T15:57:46,456 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScanner/46ee52ab2a0dcd7d79aec6f76a473fd3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:46,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741857_1033 (size=5153) 2024-11-23T15:57:46,458 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:46,458 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46ee52ab2a0dcd7d79aec6f76a473fd3: Waiting for close lock at 1732377465915Running coprocessor pre-close hooks at 1732377465916 (+1 ms)Disabling compacts and flushes for region at 1732377465916Disabling writes for close at 1732377465916Obtaining lock to block concurrent updates at 1732377465921 (+5 ms)Preparing flush snapshotting stores in 46ee52ab2a0dcd7d79aec6f76a473fd3 at 1732377465921Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377465927 (+6 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. at 1732377465928 (+1 ms)Flushing 46ee52ab2a0dcd7d79aec6f76a473fd3/testFamily: creating writer at 1732377465930 (+2 ms)Flushing 46ee52ab2a0dcd7d79aec6f76a473fd3/testFamily: appending metadata at 1732377465969 (+39 ms)Flushing 46ee52ab2a0dcd7d79aec6f76a473fd3/testFamily: closing flushed file at 1732377465971 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38b51f59: reopening flushed file at 1732377466431 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46ee52ab2a0dcd7d79aec6f76a473fd3 in 528ms, sequenceid=8, compaction requested=false at 1732377466449 (+18 ms)Writing region close event to WAL at 1732377466451 (+2 ms)Running coprocessor post-close hooks at 1732377466457 (+6 ms)Closed at 1732377466458 (+1 ms) 2024-11-23T15:57:46,459 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732377423265.46ee52ab2a0dcd7d79aec6f76a473fd3. 2024-11-23T15:57:46,459 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e383d1d2091ae721c8fde2fab32043d5, disabling compactions & flushes 2024-11-23T15:57:46,459 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:46,459 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:46,459 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. after waiting 0 ms 2024-11-23T15:57:46,459 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:46,459 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e383d1d2091ae721c8fde2fab32043d5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:46,475 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/.tmp/testFamily/5bd7ff230208430c9e8f2c90a54dac49 is 61, key is row-0/testFamily:testQualifier/1732377444625/Put/seqid=0 2024-11-23T15:57:46,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741858_1034 (size=5049) 2024-11-23T15:57:46,520 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, e383d1d2091ae721c8fde2fab32043d5, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:46,720 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, e383d1d2091ae721c8fde2fab32043d5, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:46,794 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T15:57:46,795 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T15:57:46,859 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/ns/ad7d8c8bebf84967949a244772065cff 2024-11-23T15:57:46,881 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/.tmp/testFamily/5bd7ff230208430c9e8f2c90a54dac49 2024-11-23T15:57:46,890 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/.tmp/testFamily/5bd7ff230208430c9e8f2c90a54dac49 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/testFamily/5bd7ff230208430c9e8f2c90a54dac49 2024-11-23T15:57:46,890 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/table/3bdbec12914b4f20834f00956afee556 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732377400026/Put/seqid=0 2024-11-23T15:57:46,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741859_1035 (size=6699) 2024-11-23T15:57:46,898 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/testFamily/5bd7ff230208430c9e8f2c90a54dac49, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:46,899 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e383d1d2091ae721c8fde2fab32043d5 in 440ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:46,904 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e383d1d2091ae721c8fde2fab32043d5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:46,905 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:46,905 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e383d1d2091ae721c8fde2fab32043d5: Waiting for close lock at 1732377466459Running coprocessor pre-close hooks at 1732377466459Disabling compacts and flushes for region at 1732377466459Disabling writes for close at 1732377466459Obtaining lock to block concurrent updates at 1732377466459Preparing flush snapshotting stores in e383d1d2091ae721c8fde2fab32043d5 at 1732377466459Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377466460 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. at 1732377466460Flushing e383d1d2091ae721c8fde2fab32043d5/testFamily: creating writer at 1732377466460Flushing e383d1d2091ae721c8fde2fab32043d5/testFamily: appending metadata at 1732377466475 (+15 ms)Flushing e383d1d2091ae721c8fde2fab32043d5/testFamily: closing flushed file at 1732377466475Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@758b9be2: reopening flushed file at 1732377466888 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e383d1d2091ae721c8fde2fab32043d5 in 440ms, sequenceid=8, compaction requested=false at 1732377466899 (+11 ms)Writing region close event to WAL at 1732377466900 (+1 ms)Running coprocessor post-close hooks at 1732377466905 (+5 ms)Closed at 1732377466905 2024-11-23T15:57:46,905 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732377441567.e383d1d2091ae721c8fde2fab32043d5. 2024-11-23T15:57:46,905 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a034b7b0fb6a24a4005d31cccb2f9271, disabling compactions & flushes 2024-11-23T15:57:46,905 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:46,905 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:46,905 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. after waiting 0 ms 2024-11-23T15:57:46,905 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:46,905 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a034b7b0fb6a24a4005d31cccb2f9271 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:46,920 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-23T15:57:46,920 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1325): Online Regions={a034b7b0fb6a24a4005d31cccb2f9271=testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271., d2759533b2a727b251d39de591f0c199=testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199., 6d8352eae98dce633f517661da705d92=hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92., 0a8f57953b771b9b7bde5739204e9d53=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., aee6f85aed61263650d05932716e6455=testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455., fadf4a079dfa037ada244ade41166f38=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., 1588230740=hbase:meta,,1.1588230740, 690fad263b010ff3abe3855319d4fb1a=testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., 018e567d025663812ad1302286501c06=hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.} 2024-11-23T15:57:46,920 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:46,921 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/.tmp/testFamily/5d5162d15c564673b9864064f112f0e4 is 61, key is row-0/testFamily:testQualifier/1732377432424/Put/seqid=0 2024-11-23T15:57:46,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741860_1036 (size=5049) 2024-11-23T15:57:47,121 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:47,299 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/table/3bdbec12914b4f20834f00956afee556 2024-11-23T15:57:47,314 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/info/7815248bf11847ec9b3edbadaa9f1050 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/info/7815248bf11847ec9b3edbadaa9f1050 2024-11-23T15:57:47,321 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 1588230740, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, a034b7b0fb6a24a4005d31cccb2f9271, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:47,323 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/info/7815248bf11847ec9b3edbadaa9f1050, entries=100, sequenceid=56, filesize=20.1 K 2024-11-23T15:57:47,325 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/ns/ad7d8c8bebf84967949a244772065cff as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/ns/ad7d8c8bebf84967949a244772065cff 2024-11-23T15:57:47,326 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/.tmp/testFamily/5d5162d15c564673b9864064f112f0e4 2024-11-23T15:57:47,335 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/ns/ad7d8c8bebf84967949a244772065cff, entries=2, sequenceid=56, filesize=5.0 K 2024-11-23T15:57:47,336 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/.tmp/testFamily/5d5162d15c564673b9864064f112f0e4 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/testFamily/5d5162d15c564673b9864064f112f0e4 2024-11-23T15:57:47,336 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/.tmp/table/3bdbec12914b4f20834f00956afee556 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/table/3bdbec12914b4f20834f00956afee556 2024-11-23T15:57:47,345 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/testFamily/5d5162d15c564673b9864064f112f0e4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:47,345 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/table/3bdbec12914b4f20834f00956afee556, entries=20, sequenceid=56, filesize=6.5 K 2024-11-23T15:57:47,346 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a034b7b0fb6a24a4005d31cccb2f9271 in 441ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:47,347 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1427ms, sequenceid=56, compaction requested=false 2024-11-23T15:57:47,352 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNextAsync/a034b7b0fb6a24a4005d31cccb2f9271/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:47,353 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-23T15:57:47,353 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:47,353 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a034b7b0fb6a24a4005d31cccb2f9271: Waiting for close lock at 1732377466905Running coprocessor pre-close hooks at 1732377466905Disabling compacts and flushes for region at 1732377466905Disabling writes for close at 1732377466905Obtaining lock to block concurrent updates at 1732377466905Preparing flush snapshotting stores in a034b7b0fb6a24a4005d31cccb2f9271 at 1732377466905Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377466906 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. at 1732377466906Flushing a034b7b0fb6a24a4005d31cccb2f9271/testFamily: creating writer at 1732377466906Flushing a034b7b0fb6a24a4005d31cccb2f9271/testFamily: appending metadata at 1732377466920 (+14 ms)Flushing a034b7b0fb6a24a4005d31cccb2f9271/testFamily: closing flushed file at 1732377466921 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b279fb2: reopening flushed file at 1732377467335 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a034b7b0fb6a24a4005d31cccb2f9271 in 441ms, sequenceid=8, compaction requested=false at 1732377467346 (+11 ms)Writing region close event to WAL at 1732377467347 (+1 ms)Running coprocessor post-close hooks at 1732377467353 (+6 ms)Closed at 1732377467353 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732377429372.a034b7b0fb6a24a4005d31cccb2f9271. 2024-11-23T15:57:47,354 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732377465918Running coprocessor pre-close hooks at 1732377465918Disabling compacts and flushes for region at 1732377465918Disabling writes for close at 1732377465919 (+1 ms)Obtaining lock to block concurrent updates at 1732377465921 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732377465921Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732377465927 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732377465928 (+1 ms)Flushing 1588230740/info: creating writer at 1732377465930 (+2 ms)Flushing 1588230740/info: appending metadata at 1732377465975 (+45 ms)Flushing 1588230740/info: closing flushed file at 1732377465975Flushing 1588230740/ns: creating writer at 1732377466431 (+456 ms)Flushing 1588230740/ns: appending metadata at 1732377466450 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732377466450Flushing 1588230740/table: creating writer at 1732377466872 (+422 ms)Flushing 1588230740/table: appending metadata at 1732377466889 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732377466889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14fa6e8b: reopening flushed file at 1732377467313 (+424 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13793b42: reopening flushed file at 1732377467323 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33f60744: reopening flushed file at 1732377467335 (+12 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1427ms, sequenceid=56, compaction requested=false at 1732377467347 (+12 ms)Writing region close event to WAL at 1732377467349 (+2 ms)Running coprocessor post-close hooks at 1732377467353 (+4 ms)Closed at 1732377467354 (+1 ms) 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2759533b2a727b251d39de591f0c199, disabling compactions & flushes 2024-11-23T15:57:47,354 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. after waiting 0 ms 2024-11-23T15:57:47,354 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:47,354 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d2759533b2a727b251d39de591f0c199 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:47,375 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/.tmp/testFamily/d9eaf56d612741238e3f9410488ef15a is 61, key is row-0/testFamily:testQualifier/1732377423208/Put/seqid=0 2024-11-23T15:57:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741861_1037 (size=5049) 2024-11-23T15:57:47,521 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:47,603 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=b712f9af2c12:34557 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-23T15:57:47,722 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, aee6f85aed61263650d05932716e6455, d2759533b2a727b251d39de591f0c199, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:47,782 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/.tmp/testFamily/d9eaf56d612741238e3f9410488ef15a 2024-11-23T15:57:47,796 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/.tmp/testFamily/d9eaf56d612741238e3f9410488ef15a as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/testFamily/d9eaf56d612741238e3f9410488ef15a 2024-11-23T15:57:47,807 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/testFamily/d9eaf56d612741238e3f9410488ef15a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:47,808 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d2759533b2a727b251d39de591f0c199 in 454ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:47,812 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextException/d2759533b2a727b251d39de591f0c199/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:47,813 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:47,813 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2759533b2a727b251d39de591f0c199: Waiting for close lock at 1732377467354Running coprocessor pre-close hooks at 1732377467354Disabling compacts and flushes for region at 1732377467354Disabling writes for close at 1732377467354Obtaining lock to block concurrent updates at 1732377467354Preparing flush snapshotting stores in d2759533b2a727b251d39de591f0c199 at 1732377467354Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377467355 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. at 1732377467355Flushing d2759533b2a727b251d39de591f0c199/testFamily: creating writer at 1732377467356 (+1 ms)Flushing d2759533b2a727b251d39de591f0c199/testFamily: appending metadata at 1732377467375 (+19 ms)Flushing d2759533b2a727b251d39de591f0c199/testFamily: closing flushed file at 1732377467375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d1ae3c: reopening flushed file at 1732377467794 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d2759533b2a727b251d39de591f0c199 in 454ms, sequenceid=8, compaction requested=false at 1732377467808 (+14 ms)Writing region close event to WAL at 1732377467809 (+1 ms)Running coprocessor post-close hooks at 1732377467813 (+4 ms)Closed at 1732377467813 2024-11-23T15:57:47,813 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732377420149.d2759533b2a727b251d39de591f0c199. 2024-11-23T15:57:47,814 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d8352eae98dce633f517661da705d92, disabling compactions & flushes 2024-11-23T15:57:47,814 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:57:47,814 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:57:47,814 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. after waiting 0 ms 2024-11-23T15:57:47,814 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:57:47,814 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6d8352eae98dce633f517661da705d92 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:47,829 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/.tmp/testFamily/b032a31b2204406782a242eaad9520e2 is 61, key is row-0/testFamily:testQualifier/1732377414108/Put/seqid=0 2024-11-23T15:57:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741862_1038 (size=5049) 2024-11-23T15:57:47,922 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-23T15:57:47,922 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1325): Online Regions={6d8352eae98dce633f517661da705d92=hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92., 0a8f57953b771b9b7bde5739204e9d53=hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., aee6f85aed61263650d05932716e6455=testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455., fadf4a079dfa037ada244ade41166f38=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., 690fad263b010ff3abe3855319d4fb1a=testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., 018e567d025663812ad1302286501c06=hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.} 2024-11-23T15:57:47,922 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:48,123 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 690fad263b010ff3abe3855319d4fb1a, 6d8352eae98dce633f517661da705d92, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:48,235 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/.tmp/testFamily/b032a31b2204406782a242eaad9520e2 2024-11-23T15:57:48,249 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/.tmp/testFamily/b032a31b2204406782a242eaad9520e2 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/testFamily/b032a31b2204406782a242eaad9520e2 2024-11-23T15:57:48,258 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/testFamily/b032a31b2204406782a242eaad9520e2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:48,259 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6d8352eae98dce633f517661da705d92 in 445ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:48,264 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNextAsync/6d8352eae98dce633f517661da705d92/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:48,265 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:57:48,265 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d8352eae98dce633f517661da705d92: Waiting for close lock at 1732377467813Running coprocessor pre-close hooks at 1732377467813Disabling compacts and flushes for region at 1732377467813Disabling writes for close at 1732377467814 (+1 ms)Obtaining lock to block concurrent updates at 1732377467814Preparing flush snapshotting stores in 6d8352eae98dce633f517661da705d92 at 1732377467814Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377467814Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. at 1732377467815 (+1 ms)Flushing 6d8352eae98dce633f517661da705d92/testFamily: creating writer at 1732377467815Flushing 6d8352eae98dce633f517661da705d92/testFamily: appending metadata at 1732377467828 (+13 ms)Flushing 6d8352eae98dce633f517661da705d92/testFamily: closing flushed file at 1732377467828Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b6f9263: reopening flushed file at 1732377468248 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6d8352eae98dce633f517661da705d92 in 445ms, sequenceid=8, compaction requested=false at 1732377468259 (+11 ms)Writing region close event to WAL at 1732377468260 (+1 ms)Running coprocessor post-close hooks at 1732377468265 (+5 ms)Closed at 1732377468265 2024-11-23T15:57:48,265 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732377411051.6d8352eae98dce633f517661da705d92. 2024-11-23T15:57:48,265 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a8f57953b771b9b7bde5739204e9d53, disabling compactions & flushes 2024-11-23T15:57:48,265 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:57:48,265 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:57:48,265 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. after waiting 0 ms 2024-11-23T15:57:48,265 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:57:48,265 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0a8f57953b771b9b7bde5739204e9d53 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:48,281 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/.tmp/testFamily/36885585f47d4dc38744ca52dbacdb79 is 61, key is row-0/testFamily:testQualifier/1732377405006/Put/seqid=0 2024-11-23T15:57:48,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741863_1039 (size=5049) 2024-11-23T15:57:48,323 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 690fad263b010ff3abe3855319d4fb1a, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:48,524 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 0a8f57953b771b9b7bde5739204e9d53, 690fad263b010ff3abe3855319d4fb1a, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:48,688 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/.tmp/testFamily/36885585f47d4dc38744ca52dbacdb79 2024-11-23T15:57:48,703 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/.tmp/testFamily/36885585f47d4dc38744ca52dbacdb79 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/testFamily/36885585f47d4dc38744ca52dbacdb79 2024-11-23T15:57:48,711 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/testFamily/36885585f47d4dc38744ca52dbacdb79, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:48,712 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0a8f57953b771b9b7bde5739204e9d53 in 447ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:48,717 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnOpenScanner/0a8f57953b771b9b7bde5739204e9d53/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:48,718 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:57:48,718 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a8f57953b771b9b7bde5739204e9d53: Waiting for close lock at 1732377468265Running coprocessor pre-close hooks at 1732377468265Disabling compacts and flushes for region at 1732377468265Disabling writes for close at 1732377468265Obtaining lock to block concurrent updates at 1732377468265Preparing flush snapshotting stores in 0a8f57953b771b9b7bde5739204e9d53 at 1732377468265Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377468265Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. at 1732377468266 (+1 ms)Flushing 0a8f57953b771b9b7bde5739204e9d53/testFamily: creating writer at 1732377468266Flushing 0a8f57953b771b9b7bde5739204e9d53/testFamily: appending metadata at 1732377468280 (+14 ms)Flushing 0a8f57953b771b9b7bde5739204e9d53/testFamily: closing flushed file at 1732377468280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23907314: reopening flushed file at 1732377468702 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0a8f57953b771b9b7bde5739204e9d53 in 447ms, sequenceid=8, compaction requested=false at 1732377468712 (+10 ms)Writing region close event to WAL at 1732377468713 (+1 ms)Running coprocessor post-close hooks at 1732377468718 (+5 ms)Closed at 1732377468718 2024-11-23T15:57:48,718 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732377401942.0a8f57953b771b9b7bde5739204e9d53. 2024-11-23T15:57:48,718 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aee6f85aed61263650d05932716e6455, disabling compactions & flushes 2024-11-23T15:57:48,719 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:48,719 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:48,719 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. after waiting 0 ms 2024-11-23T15:57:48,719 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:48,719 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing aee6f85aed61263650d05932716e6455 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:48,724 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:48,734 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/.tmp/testFamily/0fc4166cdad24c919de16cf789b88beb is 61, key is row-0/testFamily:testQualifier/1732377438511/Put/seqid=0 2024-11-23T15:57:48,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741864_1040 (size=5049) 2024-11-23T15:57:48,924 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-23T15:57:48,924 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1325): Online Regions={aee6f85aed61263650d05932716e6455=testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455., fadf4a079dfa037ada244ade41166f38=testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., 690fad263b010ff3abe3855319d4fb1a=testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., 018e567d025663812ad1302286501c06=hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.} 2024-11-23T15:57:48,924 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:49,125 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a, aee6f85aed61263650d05932716e6455, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:49,141 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/.tmp/testFamily/0fc4166cdad24c919de16cf789b88beb 2024-11-23T15:57:49,156 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/.tmp/testFamily/0fc4166cdad24c919de16cf789b88beb as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/testFamily/0fc4166cdad24c919de16cf789b88beb 2024-11-23T15:57:49,165 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/testFamily/0fc4166cdad24c919de16cf789b88beb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:49,166 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aee6f85aed61263650d05932716e6455 in 447ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:49,172 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnNext/aee6f85aed61263650d05932716e6455/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:49,173 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:49,174 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aee6f85aed61263650d05932716e6455: Waiting for close lock at 1732377468718Running coprocessor pre-close hooks at 1732377468718Disabling compacts and flushes for region at 1732377468718Disabling writes for close at 1732377468719 (+1 ms)Obtaining lock to block concurrent updates at 1732377468719Preparing flush snapshotting stores in aee6f85aed61263650d05932716e6455 at 1732377468719Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377468719Flushing stores of testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. at 1732377468720 (+1 ms)Flushing aee6f85aed61263650d05932716e6455/testFamily: creating writer at 1732377468720Flushing aee6f85aed61263650d05932716e6455/testFamily: appending metadata at 1732377468733 (+13 ms)Flushing aee6f85aed61263650d05932716e6455/testFamily: closing flushed file at 1732377468733Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c3db36: reopening flushed file at 1732377469155 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aee6f85aed61263650d05932716e6455 in 447ms, sequenceid=8, compaction requested=false at 1732377469166 (+11 ms)Writing region close event to WAL at 1732377469167 (+1 ms)Running coprocessor post-close hooks at 1732377469173 (+6 ms)Closed at 1732377469173 2024-11-23T15:57:49,174 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732377435467.aee6f85aed61263650d05932716e6455. 2024-11-23T15:57:49,174 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fadf4a079dfa037ada244ade41166f38, disabling compactions & flushes 2024-11-23T15:57:49,174 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:49,174 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:49,174 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. after waiting 0 ms 2024-11-23T15:57:49,174 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:49,174 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fadf4a079dfa037ada244ade41166f38 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:49,192 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/.tmp/testFamily/36d7a71bfb6b435784a874d5cd085192 is 61, key is row-0/testFamily:testQualifier/1732377462807/Put/seqid=0 2024-11-23T15:57:49,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741865_1041 (size=5049) 2024-11-23T15:57:49,325 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:49,526 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a, fadf4a079dfa037ada244ade41166f38 2024-11-23T15:57:49,599 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/.tmp/testFamily/36d7a71bfb6b435784a874d5cd085192 2024-11-23T15:57:49,614 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/.tmp/testFamily/36d7a71bfb6b435784a874d5cd085192 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/testFamily/36d7a71bfb6b435784a874d5cd085192 2024-11-23T15:57:49,621 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/testFamily/36d7a71bfb6b435784a874d5cd085192, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:49,622 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fadf4a079dfa037ada244ade41166f38 in 448ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:49,628 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testNormalScanTimeoutOnOpenScannerAsync/fadf4a079dfa037ada244ade41166f38/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:49,629 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:49,629 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fadf4a079dfa037ada244ade41166f38: Waiting for close lock at 1732377469174Running coprocessor pre-close hooks at 1732377469174Disabling compacts and flushes for region at 1732377469174Disabling writes for close at 1732377469174Obtaining lock to block concurrent updates at 1732377469174Preparing flush snapshotting stores in fadf4a079dfa037ada244ade41166f38 at 1732377469174Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377469175 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. at 1732377469175Flushing fadf4a079dfa037ada244ade41166f38/testFamily: creating writer at 1732377469175Flushing fadf4a079dfa037ada244ade41166f38/testFamily: appending metadata at 1732377469191 (+16 ms)Flushing fadf4a079dfa037ada244ade41166f38/testFamily: closing flushed file at 1732377469191Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@700e0d57: reopening flushed file at 1732377469613 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fadf4a079dfa037ada244ade41166f38 in 448ms, sequenceid=8, compaction requested=false at 1732377469622 (+9 ms)Writing region close event to WAL at 1732377469623 (+1 ms)Running coprocessor post-close hooks at 1732377469629 (+6 ms)Closed at 1732377469629 2024-11-23T15:57:49,629 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732377459755.fadf4a079dfa037ada244ade41166f38. 2024-11-23T15:57:49,629 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 690fad263b010ff3abe3855319d4fb1a, disabling compactions & flushes 2024-11-23T15:57:49,629 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:57:49,630 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:57:49,630 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. after waiting 0 ms 2024-11-23T15:57:49,630 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:57:49,630 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 690fad263b010ff3abe3855319d4fb1a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:49,650 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/.tmp/testFamily/23ddc1a8126c4d8f93dd3ccd3c0349b3 is 61, key is row-0/testFamily:testQualifier/1732377401857/Put/seqid=0 2024-11-23T15:57:49,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741866_1042 (size=5049) 2024-11-23T15:57:49,726 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:57:49,926 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T15:57:49,926 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1325): Online Regions={690fad263b010ff3abe3855319d4fb1a=testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., 018e567d025663812ad1302286501c06=hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06.} 2024-11-23T15:57:49,926 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06, 690fad263b010ff3abe3855319d4fb1a 2024-11-23T15:57:50,057 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/.tmp/testFamily/23ddc1a8126c4d8f93dd3ccd3c0349b3 2024-11-23T15:57:50,074 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/.tmp/testFamily/23ddc1a8126c4d8f93dd3ccd3c0349b3 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/testFamily/23ddc1a8126c4d8f93dd3ccd3c0349b3 2024-11-23T15:57:50,081 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/testFamily/23ddc1a8126c4d8f93dd3ccd3c0349b3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:50,082 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 690fad263b010ff3abe3855319d4fb1a in 452ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:50,087 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/default/testRetryOutOfOrderScannerNextExceptionAsync/690fad263b010ff3abe3855319d4fb1a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:50,087 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:57:50,087 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 690fad263b010ff3abe3855319d4fb1a: Waiting for close lock at 1732377469629Running coprocessor pre-close hooks at 1732377469629Disabling compacts and flushes for region at 1732377469629Disabling writes for close at 1732377469630 (+1 ms)Obtaining lock to block concurrent updates at 1732377469630Preparing flush snapshotting stores in 690fad263b010ff3abe3855319d4fb1a at 1732377469630Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377469630Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. at 1732377469631 (+1 ms)Flushing 690fad263b010ff3abe3855319d4fb1a/testFamily: creating writer at 1732377469631Flushing 690fad263b010ff3abe3855319d4fb1a/testFamily: appending metadata at 1732377469650 (+19 ms)Flushing 690fad263b010ff3abe3855319d4fb1a/testFamily: closing flushed file at 1732377469650Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b0d7339: reopening flushed file at 1732377470072 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 690fad263b010ff3abe3855319d4fb1a in 452ms, sequenceid=8, compaction requested=false at 1732377470082 (+10 ms)Writing region close event to WAL at 1732377470083 (+1 ms)Running coprocessor post-close hooks at 1732377470087 (+4 ms)Closed at 1732377470087 2024-11-23T15:57:50,087 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732377398765.690fad263b010ff3abe3855319d4fb1a. 2024-11-23T15:57:50,088 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 018e567d025663812ad1302286501c06, disabling compactions & flushes 2024-11-23T15:57:50,088 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:50,088 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:50,088 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. after waiting 0 ms 2024-11-23T15:57:50,088 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:50,088 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 018e567d025663812ad1302286501c06 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T15:57:50,103 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/.tmp/testFamily/057bf7c7b56741fbb4605e83abf709aa is 61, key is row-0/testFamily:testQualifier/1732377453706/Put/seqid=0 2024-11-23T15:57:50,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741867_1043 (size=5049) 2024-11-23T15:57:50,127 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06 2024-11-23T15:57:50,327 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06 2024-11-23T15:57:50,510 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/.tmp/testFamily/057bf7c7b56741fbb4605e83abf709aa 2024-11-23T15:57:50,523 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/.tmp/testFamily/057bf7c7b56741fbb4605e83abf709aa as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/testFamily/057bf7c7b56741fbb4605e83abf709aa 2024-11-23T15:57:50,527 DEBUG [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1351): Waiting on 018e567d025663812ad1302286501c06 2024-11-23T15:57:50,530 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/testFamily/057bf7c7b56741fbb4605e83abf709aa, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T15:57:50,531 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 018e567d025663812ad1302286501c06 in 443ms, sequenceid=8, compaction requested=false 2024-11-23T15:57:50,536 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/data/hbase/testMetaScanTimeoutOnNext/018e567d025663812ad1302286501c06/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T15:57:50,536 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:50,537 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 018e567d025663812ad1302286501c06: Waiting for close lock at 1732377470088Running coprocessor pre-close hooks at 1732377470088Disabling compacts and flushes for region at 1732377470088Disabling writes for close at 1732377470088Obtaining lock to block concurrent updates at 1732377470088Preparing flush snapshotting stores in 018e567d025663812ad1302286501c06 at 1732377470088Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732377470088Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. at 1732377470089 (+1 ms)Flushing 018e567d025663812ad1302286501c06/testFamily: creating writer at 1732377470089Flushing 018e567d025663812ad1302286501c06/testFamily: appending metadata at 1732377470102 (+13 ms)Flushing 018e567d025663812ad1302286501c06/testFamily: closing flushed file at 1732377470102Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44131565: reopening flushed file at 1732377470522 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 018e567d025663812ad1302286501c06 in 443ms, sequenceid=8, compaction requested=false at 1732377470531 (+9 ms)Writing region close event to WAL at 1732377470532 (+1 ms)Running coprocessor post-close hooks at 1732377470536 (+4 ms)Closed at 1732377470536 2024-11-23T15:57:50,537 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732377450664.018e567d025663812ad1302286501c06. 2024-11-23T15:57:50,728 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(976): stopping server b712f9af2c12,41227,1732377334860; all regions closed. 2024-11-23T15:57:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741834_1010 (size=19479) 2024-11-23T15:57:50,746 DEBUG [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/oldWALs 2024-11-23T15:57:50,747 INFO [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C41227%2C1732377334860.meta:.meta(num 1732377398270) 2024-11-23T15:57:50,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741833_1009 (size=20031) 2024-11-23T15:57:50,755 DEBUG [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/oldWALs 2024-11-23T15:57:50,755 INFO [RS:0;b712f9af2c12:41227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C41227%2C1732377334860:(num 1732377397862) 2024-11-23T15:57:50,755 DEBUG [RS:0;b712f9af2c12:41227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:57:50,755 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:57:50,756 INFO [RS:0;b712f9af2c12:41227 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:57:50,756 INFO [RS:0;b712f9af2c12:41227 {}] hbase.ChoreService(370): Chore service for: regionserver/b712f9af2c12:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T15:57:50,756 INFO [RS:0;b712f9af2c12:41227 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:57:50,756 INFO [regionserver/b712f9af2c12:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:57:50,758 INFO [RS:0;b712f9af2c12:41227 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41227 2024-11-23T15:57:50,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b712f9af2c12,41227,1732377334860 2024-11-23T15:57:50,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:57:50,821 INFO [RS:0;b712f9af2c12:41227 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:57:50,832 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b712f9af2c12,41227,1732377334860] 2024-11-23T15:57:50,832 INFO [regionserver/b712f9af2c12:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:57:50,840 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b712f9af2c12,41227,1732377334860 already deleted, retry=false 2024-11-23T15:57:50,840 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b712f9af2c12,41227,1732377334860 expired; onlineServers=0 2024-11-23T15:57:50,841 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b712f9af2c12,34557,1732377334154' ***** 2024-11-23T15:57:50,841 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T15:57:50,841 INFO [M:0;b712f9af2c12:34557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:57:50,842 INFO [M:0;b712f9af2c12:34557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:57:50,842 DEBUG [M:0;b712f9af2c12:34557 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T15:57:50,842 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T15:57:50,842 DEBUG [M:0;b712f9af2c12:34557 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T15:57:50,842 DEBUG [master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732377337657 {}] cleaner.HFileCleaner(306): Exit Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732377337657,5,FailOnTimeoutGroup] 2024-11-23T15:57:50,842 DEBUG [master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732377337656 {}] cleaner.HFileCleaner(306): Exit Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732377337656,5,FailOnTimeoutGroup] 2024-11-23T15:57:50,843 INFO [M:0;b712f9af2c12:34557 {}] hbase.ChoreService(370): Chore service for: master/b712f9af2c12:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T15:57:50,843 INFO [M:0;b712f9af2c12:34557 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:57:50,843 DEBUG [M:0;b712f9af2c12:34557 {}] master.HMaster(1795): Stopping service threads 2024-11-23T15:57:50,843 INFO [M:0;b712f9af2c12:34557 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T15:57:50,843 INFO [M:0;b712f9af2c12:34557 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T15:57:50,844 INFO [M:0;b712f9af2c12:34557 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T15:57:50,844 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T15:57:50,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T15:57:50,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:57:50,854 DEBUG [M:0;b712f9af2c12:34557 {}] zookeeper.ZKUtil(347): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T15:57:50,854 WARN [M:0;b712f9af2c12:34557 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T15:57:50,857 INFO [M:0;b712f9af2c12:34557 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/.lastflushedseqids 2024-11-23T15:57:50,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741868_1044 (size=649) 2024-11-23T15:57:50,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:57:50,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41227-0x10169b1e0db0001, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:57:50,933 INFO [RS:0;b712f9af2c12:41227 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:57:50,933 INFO [RS:0;b712f9af2c12:41227 {}] regionserver.HRegionServer(1031): Exiting; stopping=b712f9af2c12,41227,1732377334860; zookeeper connection closed. 2024-11-23T15:57:50,934 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e61832d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e61832d 2024-11-23T15:57:50,935 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T15:57:51,274 INFO [M:0;b712f9af2c12:34557 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T15:57:51,275 INFO [M:0;b712f9af2c12:34557 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T15:57:51,275 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T15:57:51,276 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:57:51,276 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:57:51,276 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T15:57:51,276 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:57:51,277 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.99 KB heapSize=190.89 KB 2024-11-23T15:57:51,298 DEBUG [M:0;b712f9af2c12:34557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a10a81e6ab164fe084e66aea12c6c8d7 is 82, key is hbase:meta,,1/info:regioninfo/1732377398353/Put/seqid=0 2024-11-23T15:57:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741869_1045 (size=5672) 2024-11-23T15:57:51,706 INFO [M:0;b712f9af2c12:34557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a10a81e6ab164fe084e66aea12c6c8d7 2024-11-23T15:57:51,734 DEBUG [M:0;b712f9af2c12:34557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f74b2c5a7af4b228f51dadbc1a8ec80 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732377400033/Put/seqid=0 2024-11-23T15:57:51,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741870_1046 (size=14070) 2024-11-23T15:57:51,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:57:52,140 INFO [M:0;b712f9af2c12:34557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f74b2c5a7af4b228f51dadbc1a8ec80 2024-11-23T15:57:52,147 INFO [M:0;b712f9af2c12:34557 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f74b2c5a7af4b228f51dadbc1a8ec80 2024-11-23T15:57:52,162 DEBUG [M:0;b712f9af2c12:34557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e47254808f1244cba1942fc572eed6eb is 69, key is b712f9af2c12,41227,1732377334860/rs:state/1732377367685/Put/seqid=0 2024-11-23T15:57:52,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741871_1047 (size=5156) 2024-11-23T15:57:52,568 INFO [M:0;b712f9af2c12:34557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e47254808f1244cba1942fc572eed6eb 2024-11-23T15:57:52,581 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a10a81e6ab164fe084e66aea12c6c8d7 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a10a81e6ab164fe084e66aea12c6c8d7 2024-11-23T15:57:52,589 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a10a81e6ab164fe084e66aea12c6c8d7, entries=8, sequenceid=317, filesize=5.5 K 2024-11-23T15:57:52,590 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f74b2c5a7af4b228f51dadbc1a8ec80 as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f74b2c5a7af4b228f51dadbc1a8ec80 2024-11-23T15:57:52,597 INFO [M:0;b712f9af2c12:34557 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f74b2c5a7af4b228f51dadbc1a8ec80 2024-11-23T15:57:52,598 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f74b2c5a7af4b228f51dadbc1a8ec80, entries=33, sequenceid=317, filesize=13.7 K 2024-11-23T15:57:52,599 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e47254808f1244cba1942fc572eed6eb as hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e47254808f1244cba1942fc572eed6eb 2024-11-23T15:57:52,605 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e47254808f1244cba1942fc572eed6eb, entries=1, sequenceid=317, filesize=5.0 K 2024-11-23T15:57:52,607 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.99 KB/166903, heapSize ~190.59 KB/195168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1330ms, sequenceid=317, compaction requested=false 2024-11-23T15:57:52,608 INFO [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:57:52,608 DEBUG [M:0;b712f9af2c12:34557 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732377471275Disabling compacts and flushes for region at 1732377471275Disabling writes for close at 1732377471276 (+1 ms)Obtaining lock to block concurrent updates at 1732377471277 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732377471277Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166903, getHeapSize=195408, getOffHeapSize=0, getCellsCount=364 at 1732377471278 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732377471280 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732377471280Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732377471298 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732377471298Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732377471718 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732377471734 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732377471734Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732377472147 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732377472162 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732377472162Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b628666: reopening flushed file at 1732377472579 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@735636b0: reopening flushed file at 1732377472589 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13260229: reopening flushed file at 1732377472598 (+9 ms)Finished flush of dataSize ~162.99 KB/166903, heapSize ~190.59 KB/195168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1330ms, sequenceid=317, compaction requested=false at 1732377472607 (+9 ms)Writing region close event to WAL at 1732377472608 (+1 ms)Closed at 1732377472608 2024-11-23T15:57:52,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32865 is added to blk_1073741830_1006 (size=188115) 2024-11-23T15:57:52,611 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/90d324e4-2b40-d41a-840f-d81054e4cb61/MasterData/WALs/b712f9af2c12,34557,1732377334154/b712f9af2c12%2C34557%2C1732377334154.1732377336994 not finished, retry = 0 2024-11-23T15:57:52,713 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:57:52,713 INFO [M:0;b712f9af2c12:34557 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T15:57:52,713 INFO [M:0;b712f9af2c12:34557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34557 2024-11-23T15:57:52,714 INFO [M:0;b712f9af2c12:34557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:57:52,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:57:52,857 INFO [M:0;b712f9af2c12:34557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:57:52,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34557-0x10169b1e0db0000, quorum=127.0.0.1:52376, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:57:52,865 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc78d53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:57:52,867 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:57:52,867 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:57:52,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:57:52,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/hadoop.log.dir/,STOPPED} 2024-11-23T15:57:52,869 WARN [BP-942193177-172.17.0.2-1732377330606 heartbeating to localhost/127.0.0.1:39973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T15:57:52,869 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T15:57:52,869 WARN [BP-942193177-172.17.0.2-1732377330606 heartbeating to localhost/127.0.0.1:39973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942193177-172.17.0.2-1732377330606 (Datanode Uuid 19145e33-d8d6-459d-8a4a-15be26ca102e) service to localhost/127.0.0.1:39973 2024-11-23T15:57:52,869 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T15:57:52,871 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/data/data1/current/BP-942193177-172.17.0.2-1732377330606 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:57:52,871 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/cluster_3fc0d40a-f37e-b79e-fa6d-f80a562a94b9/data/data2/current/BP-942193177-172.17.0.2-1732377330606 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:57:52,871 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T15:57:52,878 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@562f0d0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T15:57:52,879 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:57:52,879 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:57:52,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:57:52,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95a4e8c6-7d19-ead5-01aa-8f5c8e7f6bb5/hadoop.log.dir/,STOPPED} 2024-11-23T15:57:52,887 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T15:57:52,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down