2024-11-24 17:36:20,961 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-24 17:36:20,981 main DEBUG Took 0.017391 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 17:36:20,981 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 17:36:20,982 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 17:36:20,983 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 17:36:20,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:20,995 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 17:36:21,012 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,018 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,019 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,021 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,023 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,024 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,025 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,026 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:36:21,028 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,028 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 17:36:21,030 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:36:21,032 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 17:36:21,034 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 17:36:21,034 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 17:36:21,035 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 17:36:21,036 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 17:36:21,043 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 17:36:21,046 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 17:36:21,047 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 17:36:21,048 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 17:36:21,048 main DEBUG createAppenders(={Console}) 2024-11-24 17:36:21,049 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-24 17:36:21,049 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-24 17:36:21,049 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-24 17:36:21,050 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 17:36:21,050 main DEBUG OutputStream closed 2024-11-24 17:36:21,050 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 17:36:21,050 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 17:36:21,051 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-24 17:36:21,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 17:36:21,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 17:36:21,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 17:36:21,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 17:36:21,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 17:36:21,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 17:36:21,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 17:36:21,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 17:36:21,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 17:36:21,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 17:36:21,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 17:36:21,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 17:36:21,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 17:36:21,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 17:36:21,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 17:36:21,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 17:36:21,125 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 17:36:21,126 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 17:36:21,128 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 17:36:21,129 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-24 17:36:21,129 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 17:36:21,130 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-24T17:36:21,143 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-24 17:36:21,146 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 17:36:21,146 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T17:36:21,402 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1 2024-11-24T17:36:21,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T17:36:21,437 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3, deleteOnExit=true 2024-11-24T17:36:21,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T17:36:21,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/test.cache.data in system properties and HBase conf 2024-11-24T17:36:21,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T17:36:21,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/hadoop.log.dir in system properties and HBase conf 2024-11-24T17:36:21,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T17:36:21,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T17:36:21,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T17:36:21,549 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T17:36:21,639 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T17:36:21,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T17:36:21,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T17:36:21,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T17:36:21,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T17:36:21,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T17:36:21,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T17:36:21,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T17:36:21,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T17:36:21,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T17:36:21,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/nfs.dump.dir in system properties and HBase conf 2024-11-24T17:36:21,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/java.io.tmpdir in system properties and HBase conf 2024-11-24T17:36:21,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T17:36:21,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T17:36:21,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T17:36:22,718 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T17:36:22,790 INFO [Time-limited test {}] log.Log(170): Logging initialized @2480ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T17:36:22,858 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:36:22,919 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T17:36:22,940 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T17:36:22,940 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T17:36:22,942 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T17:36:22,954 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:36:22,958 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3bcd68eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/hadoop.log.dir/,AVAILABLE} 2024-11-24T17:36:22,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3959e81b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T17:36:23,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ce6ffde{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/java.io.tmpdir/jetty-localhost-40383-hadoop-hdfs-3_4_1-tests_jar-_-any-12630012443616609617/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T17:36:23,151 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5c90255f{HTTP/1.1, (http/1.1)}{localhost:40383} 2024-11-24T17:36:23,151 INFO [Time-limited test {}] server.Server(415): Started @2843ms 2024-11-24T17:36:23,785 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:36:23,793 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T17:36:23,793 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T17:36:23,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T17:36:23,794 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T17:36:23,794 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f7444f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/hadoop.log.dir/,AVAILABLE} 2024-11-24T17:36:23,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30fd9473{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T17:36:23,894 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bc105ee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/java.io.tmpdir/jetty-localhost-34639-hadoop-hdfs-3_4_1-tests_jar-_-any-15124273683617095357/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T17:36:23,894 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@360caa46{HTTP/1.1, (http/1.1)}{localhost:34639} 2024-11-24T17:36:23,895 INFO [Time-limited test {}] server.Server(415): Started @3586ms 2024-11-24T17:36:23,942 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T17:36:24,933 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/data/data2/current/BP-500573805-172.17.0.3-1732469782164/current, will proceed with Du for space computation calculation, 2024-11-24T17:36:24,933 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/data/data1/current/BP-500573805-172.17.0.3-1732469782164/current, will proceed with Du for space computation calculation, 2024-11-24T17:36:24,959 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T17:36:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7113fb154e2a8cb with lease ID 0x3372c2d0842e2801: Processing first storage report for DS-341e13c5-ee4f-48ca-a5ae-b121d6c8f17c from datanode DatanodeRegistration(127.0.0.1:36247, datanodeUuid=df4dbb39-e07a-4574-bd92-bc76e4ec5c2e, infoPort=41977, infoSecurePort=0, ipcPort=39135, storageInfo=lv=-57;cid=testClusterID;nsid=1649399093;c=1732469782164) 2024-11-24T17:36:25,003 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7113fb154e2a8cb with lease ID 0x3372c2d0842e2801: from storage DS-341e13c5-ee4f-48ca-a5ae-b121d6c8f17c node DatanodeRegistration(127.0.0.1:36247, datanodeUuid=df4dbb39-e07a-4574-bd92-bc76e4ec5c2e, infoPort=41977, infoSecurePort=0, ipcPort=39135, storageInfo=lv=-57;cid=testClusterID;nsid=1649399093;c=1732469782164), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T17:36:25,004 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7113fb154e2a8cb with lease ID 0x3372c2d0842e2801: Processing first storage report for DS-4fd111d8-0534-4155-a5a2-b3b9b531fb94 from datanode DatanodeRegistration(127.0.0.1:36247, datanodeUuid=df4dbb39-e07a-4574-bd92-bc76e4ec5c2e, infoPort=41977, infoSecurePort=0, ipcPort=39135, storageInfo=lv=-57;cid=testClusterID;nsid=1649399093;c=1732469782164) 2024-11-24T17:36:25,004 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7113fb154e2a8cb with lease ID 0x3372c2d0842e2801: from storage DS-4fd111d8-0534-4155-a5a2-b3b9b531fb94 node DatanodeRegistration(127.0.0.1:36247, datanodeUuid=df4dbb39-e07a-4574-bd92-bc76e4ec5c2e, infoPort=41977, infoSecurePort=0, ipcPort=39135, storageInfo=lv=-57;cid=testClusterID;nsid=1649399093;c=1732469782164), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T17:36:25,062 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1 2024-11-24T17:36:25,128 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/zookeeper_0, clientPort=59932, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T17:36:25,137 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59932 2024-11-24T17:36:25,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:25,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:25,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741825_1001 (size=7) 2024-11-24T17:36:25,745 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e with version=8 2024-11-24T17:36:25,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/hbase-staging 2024-11-24T17:36:25,824 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T17:36:26,143 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a23eb99f71f0:0 server-side Connection retries=9 2024-11-24T17:36:26,152 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:36:26,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T17:36:26,157 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T17:36:26,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:36:26,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T17:36:26,293 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T17:36:26,344 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T17:36:26,352 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T17:36:26,355 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T17:36:26,377 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 106370 (auto-detected) 2024-11-24T17:36:26,378 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-24T17:36:26,394 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33353 2024-11-24T17:36:26,414 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33353 connecting to ZooKeeper ensemble=127.0.0.1:59932 2024-11-24T17:36:26,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333530x0, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T17:36:26,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33353-0x1016deafdc30000 connected 2024-11-24T17:36:26,653 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:26,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:26,665 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:36:26,668 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e, hbase.cluster.distributed=false 2024-11-24T17:36:26,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T17:36:26,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33353 2024-11-24T17:36:26,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33353 2024-11-24T17:36:26,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33353 2024-11-24T17:36:26,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33353 2024-11-24T17:36:26,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33353 2024-11-24T17:36:26,820 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a23eb99f71f0:0 server-side Connection retries=9 2024-11-24T17:36:26,823 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:36:26,823 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T17:36:26,823 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T17:36:26,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:36:26,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T17:36:26,827 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T17:36:26,831 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T17:36:26,832 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34897 2024-11-24T17:36:26,835 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34897 connecting to ZooKeeper ensemble=127.0.0.1:59932 2024-11-24T17:36:26,837 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:26,842 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:26,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348970x0, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T17:36:26,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:348970x0, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:36:26,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34897-0x1016deafdc30001 connected 2024-11-24T17:36:26,863 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T17:36:26,869 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T17:36:26,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T17:36:26,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T17:36:26,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34897 2024-11-24T17:36:26,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34897 2024-11-24T17:36:26,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34897 2024-11-24T17:36:26,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34897 2024-11-24T17:36:26,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34897 2024-11-24T17:36:26,894 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a23eb99f71f0:33353 2024-11-24T17:36:26,895 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:26,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:36:26,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:36:26,913 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:26,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T17:36:26,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:26,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:26,946 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T17:36:26,947 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a23eb99f71f0,33353,1732469785982 from backup master directory 2024-11-24T17:36:26,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:26,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:36:26,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:36:26,957 WARN [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T17:36:26,957 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:26,959 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T17:36:26,960 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T17:36:27,017 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/hbase.id] with ID: c21d4ea2-d4d6-4b32-bc06-f004fe826b98 2024-11-24T17:36:27,018 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/.tmp/hbase.id 2024-11-24T17:36:27,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741826_1002 (size=42) 2024-11-24T17:36:27,434 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/.tmp/hbase.id]:[hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/hbase.id] 2024-11-24T17:36:27,488 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:36:27,493 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T17:36:27,511 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T17:36:27,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:27,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:27,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741827_1003 (size=196) 2024-11-24T17:36:28,004 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:36:28,007 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T17:36:28,022 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:36:28,025 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:36:28,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741828_1004 (size=1189) 2024-11-24T17:36:28,483 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store 2024-11-24T17:36:28,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741829_1005 (size=34) 2024-11-24T17:36:28,913 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T17:36:28,917 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:36:28,918 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T17:36:28,918 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:36:28,918 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:36:28,920 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T17:36:28,920 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:36:28,920 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:36:28,921 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732469788918Disabling compacts and flushes for region at 1732469788918Disabling writes for close at 1732469788920 (+2 ms)Writing region close event to WAL at 1732469788920Closed at 1732469788920 2024-11-24T17:36:28,923 WARN [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/.initializing 2024-11-24T17:36:28,923 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/WALs/a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:28,931 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T17:36:28,946 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C33353%2C1732469785982, suffix=, logDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/WALs/a23eb99f71f0,33353,1732469785982, archiveDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/oldWALs, maxLogs=10 2024-11-24T17:36:28,968 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/WALs/a23eb99f71f0,33353,1732469785982/a23eb99f71f0%2C33353%2C1732469785982.1732469788950, exclude list is [], retry=0 2024-11-24T17:36:28,984 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36247,DS-341e13c5-ee4f-48ca-a5ae-b121d6c8f17c,DISK] 2024-11-24T17:36:28,986 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T17:36:29,023 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/WALs/a23eb99f71f0,33353,1732469785982/a23eb99f71f0%2C33353%2C1732469785982.1732469788950 2024-11-24T17:36:29,024 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41977:41977)] 2024-11-24T17:36:29,024 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:36:29,025 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:36:29,027 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,028 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T17:36:29,086 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:29,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:36:29,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T17:36:29,093 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:29,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:36:29,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T17:36:29,098 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:29,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:36:29,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T17:36:29,102 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:29,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:36:29,104 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,108 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,110 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,117 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,118 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,122 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:36:29,127 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:36:29,133 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:36:29,134 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61254342, jitterRate=-0.08723917603492737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:36:29,141 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732469789039Initializing all the Stores at 1732469789041 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469789042 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469789042Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469789043 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469789043Cleaning up temporary data from old regions at 1732469789118 (+75 ms)Region opened successfully at 1732469789141 (+23 ms) 2024-11-24T17:36:29,143 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T17:36:29,179 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abcdba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a23eb99f71f0/172.17.0.3:0 2024-11-24T17:36:29,216 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T17:36:29,226 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T17:36:29,227 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T17:36:29,229 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:36:29,231 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T17:36:29,235 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T17:36:29,235 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T17:36:29,258 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T17:36:29,266 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T17:36:29,345 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T17:36:29,349 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T17:36:29,351 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T17:36:29,363 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T17:36:29,366 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T17:36:29,372 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T17:36:29,384 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T17:36:29,386 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T17:36:29,397 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T17:36:29,419 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T17:36:29,428 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T17:36:29,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T17:36:29,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T17:36:29,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:29,443 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a23eb99f71f0,33353,1732469785982, sessionid=0x1016deafdc30000, setting cluster-up flag (Was=false) 2024-11-24T17:36:29,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:29,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:29,502 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T17:36:29,506 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:29,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:29,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:36:29,566 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T17:36:29,571 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a23eb99f71f0,33353,1732469785982 2024-11-24T17:36:29,580 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T17:36:29,587 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(746): ClusterId : c21d4ea2-d4d6-4b32-bc06-f004fe826b98 2024-11-24T17:36:29,590 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T17:36:29,599 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T17:36:29,599 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T17:36:29,609 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T17:36:29,610 DEBUG [RS:0;a23eb99f71f0:34897 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ad40cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a23eb99f71f0/172.17.0.3:0 2024-11-24T17:36:29,623 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a23eb99f71f0:34897 2024-11-24T17:36:29,626 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T17:36:29,626 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T17:36:29,626 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T17:36:29,629 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(2659): reportForDuty to master=a23eb99f71f0,33353,1732469785982 with port=34897, startcode=1732469786783 2024-11-24T17:36:29,639 DEBUG [RS:0;a23eb99f71f0:34897 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T17:36:29,656 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:36:29,665 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T17:36:29,672 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T17:36:29,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43349, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T17:36:29,678 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33353 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T17:36:29,679 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a23eb99f71f0,33353,1732469785982 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T17:36:29,687 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:36:29,687 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:36:29,687 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:36:29,687 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:36:29,687 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a23eb99f71f0:0, corePoolSize=10, maxPoolSize=10 2024-11-24T17:36:29,688 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:29,688 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=2, maxPoolSize=2 2024-11-24T17:36:29,688 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:29,690 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732469819689 2024-11-24T17:36:29,691 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T17:36:29,692 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T17:36:29,693 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T17:36:29,694 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T17:36:29,695 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T17:36:29,696 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T17:36:29,696 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T17:36:29,696 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T17:36:29,697 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:29,700 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T17:36:29,701 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T17:36:29,701 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:29,701 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T17:36:29,701 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T17:36:29,703 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T17:36:29,703 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T17:36:29,704 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.large.0-1732469789704,5,FailOnTimeoutGroup] 2024-11-24T17:36:29,705 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.small.0-1732469789705,5,FailOnTimeoutGroup] 2024-11-24T17:36:29,705 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:29,705 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T17:36:29,707 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:29,707 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:29,709 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T17:36:29,709 WARN [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-24T17:36:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741831_1007 (size=1321) 2024-11-24T17:36:30,113 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T17:36:30,113 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:36:30,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741832_1008 (size=32) 2024-11-24T17:36:30,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:36:30,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T17:36:30,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T17:36:30,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:30,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:36:30,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T17:36:30,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T17:36:30,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:30,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:36:30,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T17:36:30,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T17:36:30,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:30,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:36:30,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T17:36:30,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T17:36:30,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:36:30,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:36:30,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T17:36:30,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740 2024-11-24T17:36:30,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740 2024-11-24T17:36:30,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T17:36:30,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T17:36:30,555 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:36:30,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T17:36:30,562 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:36:30,563 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62761514, jitterRate=-0.06478056311607361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:36:30,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732469790531Initializing all the Stores at 1732469790533 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469790533Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469790533Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469790534 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469790534Cleaning up temporary data from old regions at 1732469790554 (+20 ms)Region opened successfully at 1732469790567 (+13 ms) 2024-11-24T17:36:30,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T17:36:30,568 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T17:36:30,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T17:36:30,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T17:36:30,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T17:36:30,570 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T17:36:30,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732469790568Disabling compacts and flushes for region at 1732469790568Disabling writes for close at 1732469790568Writing region close event to WAL at 1732469790569 (+1 ms)Closed at 1732469790570 (+1 ms) 2024-11-24T17:36:30,573 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T17:36:30,574 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T17:36:30,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T17:36:30,586 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:36:30,589 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T17:36:30,741 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:31,742 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:32,743 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:33,744 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:34,745 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:35,746 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:36,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T17:36:36,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T17:36:36,747 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:37,748 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:38,748 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:39,749 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:40,750 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:41,751 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:42,752 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:43,753 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:44,754 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:45,755 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:46,756 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:47,757 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:48,758 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:49,760 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:50,761 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:51,762 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:52,763 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:53,764 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:54,765 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:55,058 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T17:36:55,766 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:56,767 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:57,768 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:58,769 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:59,712 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(2659): reportForDuty to master=a23eb99f71f0,33353,1732469785982 with port=34897, startcode=1732469786783 2024-11-24T17:36:59,719 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33353 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a23eb99f71f0,34897,1732469786783 2024-11-24T17:36:59,724 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33353 {}] master.ServerManager(517): Registering regionserver=a23eb99f71f0,34897,1732469786783 2024-11-24T17:36:59,731 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:36:59,731 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38651 2024-11-24T17:36:59,731 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T17:36:59,770 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:36:59,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T17:36:59,830 DEBUG [RS:0;a23eb99f71f0:34897 {}] zookeeper.ZKUtil(111): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a23eb99f71f0,34897,1732469786783 2024-11-24T17:36:59,831 WARN [RS:0;a23eb99f71f0:34897 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T17:36:59,832 INFO [RS:0;a23eb99f71f0:34897 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:36:59,832 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783 2024-11-24T17:36:59,834 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a23eb99f71f0,34897,1732469786783] 2024-11-24T17:36:59,859 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T17:36:59,870 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T17:36:59,874 INFO [RS:0;a23eb99f71f0:34897 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T17:36:59,874 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,875 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T17:36:59,882 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T17:36:59,884 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,884 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,885 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,885 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,885 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,885 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,886 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0, corePoolSize=2, maxPoolSize=2 2024-11-24T17:36:59,886 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,886 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,886 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,886 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,886 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,887 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:36:59,887 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:36:59,887 DEBUG [RS:0;a23eb99f71f0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:36:59,888 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,888 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,889 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,889 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,889 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,889 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,34897,1732469786783-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T17:36:59,907 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T17:36:59,909 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,34897,1732469786783-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,910 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,910 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.Replication(171): a23eb99f71f0,34897,1732469786783 started 2024-11-24T17:36:59,927 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:36:59,928 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1482): Serving as a23eb99f71f0,34897,1732469786783, RpcServer on a23eb99f71f0/172.17.0.3:34897, sessionid=0x1016deafdc30001 2024-11-24T17:36:59,928 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T17:36:59,929 DEBUG [RS:0;a23eb99f71f0:34897 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a23eb99f71f0,34897,1732469786783 2024-11-24T17:36:59,929 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,34897,1732469786783' 2024-11-24T17:36:59,929 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T17:36:59,930 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T17:36:59,931 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T17:36:59,931 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T17:36:59,931 DEBUG [RS:0;a23eb99f71f0:34897 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a23eb99f71f0,34897,1732469786783 2024-11-24T17:36:59,931 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,34897,1732469786783' 2024-11-24T17:36:59,931 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T17:36:59,932 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T17:36:59,932 DEBUG [RS:0;a23eb99f71f0:34897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T17:36:59,932 INFO [RS:0;a23eb99f71f0:34897 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T17:36:59,933 INFO [RS:0;a23eb99f71f0:34897 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T17:37:00,771 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:01,772 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:02,773 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:03,774 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:04,775 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:05,776 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:06,777 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:07,778 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:08,779 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:09,780 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:10,781 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:11,782 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:12,783 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:13,784 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:14,785 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:15,787 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:16,788 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:17,789 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:18,791 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:19,792 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:20,793 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:21,794 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:22,795 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:23,796 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:24,797 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:25,058 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T17:37:25,798 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:26,800 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:27,801 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:28,802 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:29,803 WARN [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:37:29,945 INFO [RS:0;a23eb99f71f0:34897 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T17:37:29,950 INFO [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C34897%2C1732469786783, suffix=, logDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783, archiveDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/oldWALs, maxLogs=32 2024-11-24T17:37:29,966 DEBUG [RS:0;a23eb99f71f0:34897 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783/a23eb99f71f0%2C34897%2C1732469786783.1732469849953, exclude list is [], retry=0 2024-11-24T17:37:29,970 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36247,DS-341e13c5-ee4f-48ca-a5ae-b121d6c8f17c,DISK] 2024-11-24T17:37:29,974 INFO [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783/a23eb99f71f0%2C34897%2C1732469786783.1732469849953 2024-11-24T17:37:29,974 DEBUG [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41977:41977)] 2024-11-24T17:37:30,058 DEBUG [a23eb99f71f0:33353 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T17:37:30,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:30,077 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34897,1732469786783, state=OPENING 2024-11-24T17:37:30,150 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T17:37:30,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:37:30,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:37:30,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:37:30,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:37:30,167 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:37:30,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:37:30,351 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T17:37:30,354 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38929, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T17:37:30,366 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T17:37:30,366 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:37:30,367 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T17:37:30,370 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C34897%2C1732469786783.meta, suffix=.meta, logDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783, archiveDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/oldWALs, maxLogs=32 2024-11-24T17:37:30,387 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783/a23eb99f71f0%2C34897%2C1732469786783.meta.1732469850373.meta, exclude list is [], retry=0 2024-11-24T17:37:30,391 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36247,DS-341e13c5-ee4f-48ca-a5ae-b121d6c8f17c,DISK] 2024-11-24T17:37:30,394 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/WALs/a23eb99f71f0,34897,1732469786783/a23eb99f71f0%2C34897%2C1732469786783.meta.1732469850373.meta 2024-11-24T17:37:30,394 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41977:41977)] 2024-11-24T17:37:30,395 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:37:30,396 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T17:37:30,399 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T17:37:30,403 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T17:37:30,406 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T17:37:30,407 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:30,407 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T17:37:30,407 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T17:37:30,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T17:37:30,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T17:37:30,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:30,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:37:30,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T17:37:30,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T17:37:30,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:30,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:37:30,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T17:37:30,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T17:37:30,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:30,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:37:30,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T17:37:30,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T17:37:30,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:30,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:37:30,421 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T17:37:30,423 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740 2024-11-24T17:37:30,425 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740 2024-11-24T17:37:30,428 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T17:37:30,428 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T17:37:30,429 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:37:30,432 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T17:37:30,434 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74402179, jitterRate=0.10867886245250702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:37:30,434 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T17:37:30,435 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732469850408Writing region info on filesystem at 1732469850408Initializing all the Stores at 1732469850410 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469850410Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469850410Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469850411 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732469850411Cleaning up temporary data from old regions at 1732469850428 (+17 ms)Running coprocessor post-open hooks at 1732469850434 (+6 ms)Region opened successfully at 1732469850435 (+1 ms) 2024-11-24T17:37:30,442 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732469850342 2024-11-24T17:37:30,454 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T17:37:30,455 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T17:37:30,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:30,459 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34897,1732469786783, state=OPEN 2024-11-24T17:37:30,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:37:30,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:37:30,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:37:30,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:37:30,517 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:30,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T17:37:30,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34897,1732469786783 in 346 msec 2024-11-24T17:37:30,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T17:37:30,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 59.9470 sec 2024-11-24T17:37:30,533 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T17:37:30,533 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T17:37:30,550 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T17:37:30,552 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34897,1732469786783, seqNum=-1] 2024-11-24T17:37:30,570 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:37:30,573 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58961, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:37:30,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 0.98 sec 2024-11-24T17:37:30,594 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732469850594, completionTime=-1 2024-11-24T17:37:30,598 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T17:37:30,598 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:37:30,622 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T17:37:30,622 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732469910622 2024-11-24T17:37:30,622 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732469970622 2024-11-24T17:37:30,622 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-24T17:37:30,624 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,33353,1732469785982-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,624 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,33353,1732469785982-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,624 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,33353,1732469785982-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,626 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a23eb99f71f0:33353, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,626 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,626 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,632 DEBUG [master/a23eb99f71f0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T17:37:30,653 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.696sec 2024-11-24T17:37:30,655 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T17:37:30,657 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T17:37:30,658 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T17:37:30,658 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T17:37:30,659 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T17:37:30,660 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,33353,1732469785982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T17:37:30,660 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,33353,1732469785982-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T17:37:30,669 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T17:37:30,670 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T17:37:30,670 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,33353,1732469785982-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:37:30,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349b18c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:37:30,674 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T17:37:30,674 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T17:37:30,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a23eb99f71f0,33353,-1 for getting cluster id 2024-11-24T17:37:30,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T17:37:30,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c21d4ea2-d4d6-4b32-bc06-f004fe826b98' 2024-11-24T17:37:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T17:37:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c21d4ea2-d4d6-4b32-bc06-f004fe826b98" 2024-11-24T17:37:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5bd34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:37:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a23eb99f71f0,33353,-1] 2024-11-24T17:37:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T17:37:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:37:30,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T17:37:30,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2081155d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:37:30,700 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T17:37:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34897,1732469786783, seqNum=-1] 2024-11-24T17:37:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:37:30,711 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:37:30,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a23eb99f71f0,33353,1732469785982 2024-11-24T17:37:30,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3a8a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:37:30,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a23eb99f71f0,33353,-1 for getting cluster id 2024-11-24T17:37:30,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T17:37:30,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c21d4ea2-d4d6-4b32-bc06-f004fe826b98' 2024-11-24T17:37:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T17:37:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c21d4ea2-d4d6-4b32-bc06-f004fe826b98" 2024-11-24T17:37:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8a2617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:37:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a23eb99f71f0,33353,-1] 2024-11-24T17:37:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T17:37:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:37:30,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T17:37:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e4f460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:37:30,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=6006 2024-11-24T17:37:30,758 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T17:37:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a23eb99f71f0,33353,1732469785982 2024-11-24T17:37:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5fdd038e 2024-11-24T17:37:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T17:37:30,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T17:37:30,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:37:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-24T17:37:30,808 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:37:30,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-24T17:37:30,814 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:30,818 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:37:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:37:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741835_1011 (size=428) 2024-11-24T17:37:31,241 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33bcf1d6c44cbf423a9903914d6f35a5, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:37:31,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741836_1012 (size=79) 2024-11-24T17:37:31,651 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:31,651 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 33bcf1d6c44cbf423a9903914d6f35a5, disabling compactions & flushes 2024-11-24T17:37:31,651 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:31,651 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:31,651 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. after waiting 0 ms 2024-11-24T17:37:31,651 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:31,651 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:31,651 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33bcf1d6c44cbf423a9903914d6f35a5: Waiting for close lock at 1732469851651Disabling compacts and flushes for region at 1732469851651Disabling writes for close at 1732469851651Writing region close event to WAL at 1732469851651Closed at 1732469851651 2024-11-24T17:37:31,654 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:37:31,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732469851654"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469851654"}]},"ts":"1732469851654"} 2024-11-24T17:37:31,663 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:37:31,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:37:31,668 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469851665"}]},"ts":"1732469851665"} 2024-11-24T17:37:31,672 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-24T17:37:31,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=33bcf1d6c44cbf423a9903914d6f35a5, ASSIGN}] 2024-11-24T17:37:31,677 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=33bcf1d6c44cbf423a9903914d6f35a5, ASSIGN 2024-11-24T17:37:31,679 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=33bcf1d6c44cbf423a9903914d6f35a5, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:37:31,831 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=33bcf1d6c44cbf423a9903914d6f35a5, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:37:31,841 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=33bcf1d6c44cbf423a9903914d6f35a5, ASSIGN because future has completed 2024-11-24T17:37:31,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33bcf1d6c44cbf423a9903914d6f35a5, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:37:32,012 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:32,012 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 33bcf1d6c44cbf423a9903914d6f35a5, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:37:32,013 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,013 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:32,013 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,013 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,016 INFO [StoreOpener-33bcf1d6c44cbf423a9903914d6f35a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,018 INFO [StoreOpener-33bcf1d6c44cbf423a9903914d6f35a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33bcf1d6c44cbf423a9903914d6f35a5 columnFamilyName testFamily 2024-11-24T17:37:32,018 DEBUG [StoreOpener-33bcf1d6c44cbf423a9903914d6f35a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:32,019 INFO [StoreOpener-33bcf1d6c44cbf423a9903914d6f35a5-1 {}] regionserver.HStore(327): Store=33bcf1d6c44cbf423a9903914d6f35a5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:37:32,020 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,021 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,022 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,023 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,023 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,026 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,030 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:37:32,031 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 33bcf1d6c44cbf423a9903914d6f35a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71562962, jitterRate=0.06637123227119446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:37:32,031 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:37:32,033 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 33bcf1d6c44cbf423a9903914d6f35a5: Running coprocessor pre-open hook at 1732469852013Writing region info on filesystem at 1732469852013Initializing all the Stores at 1732469852015 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469852015Cleaning up temporary data from old regions at 1732469852023 (+8 ms)Running coprocessor post-open hooks at 1732469852031 (+8 ms)Region opened successfully at 1732469852032 (+1 ms) 2024-11-24T17:37:32,035 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5., pid=6, masterSystemTime=1732469851998 2024-11-24T17:37:32,038 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:32,038 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:37:32,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=33bcf1d6c44cbf423a9903914d6f35a5, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:32,043 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33bcf1d6c44cbf423a9903914d6f35a5, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:37:32,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T17:37:32,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 33bcf1d6c44cbf423a9903914d6f35a5, server=a23eb99f71f0,34897,1732469786783 in 203 msec 2024-11-24T17:37:32,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T17:37:32,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=33bcf1d6c44cbf423a9903914d6f35a5, ASSIGN in 376 msec 2024-11-24T17:37:32,055 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:37:32,055 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469852055"}]},"ts":"1732469852055"} 2024-11-24T17:37:32,059 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-24T17:37:32,060 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:37:32,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2620 sec 2024-11-24T17:37:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:37:33,855 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-24T17:37:33,855 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-24T17:37:33,856 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:33,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-24T17:37:33,864 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:33,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-24T17:37:33,868 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T17:37:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34897,1732469786783, seqNum=-1] 2024-11-24T17:37:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:37:33,873 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:37:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:37:33,891 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:37:33,895 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-24T17:37:33,900 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-24T17:37:33,904 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-24T17:37:33,906 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-24T17:37:33,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2479531022807465976 2024-11-24T17:37:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469856906, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:33,914 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-24T17:37:33,916 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-24T17:37:33,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:33,918 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:33,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2479531022807465974 2024-11-24T17:37:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469856918, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:33,922 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:33,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-2479531022807465973 2024-11-24T17:37:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469856922, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:33,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:33,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-2479531022807465972 2024-11-24T17:37:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469856926, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:33,930 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:33,939 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_581555434_22 at /127.0.0.1:55506 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=5963 (was 6006) 2024-11-24T17:37:33,946 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=5963 2024-11-24T17:37:33,950 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:37:33,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-24T17:37:33,954 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:37:33,954 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:33,954 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-24T17:37:33,956 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:37:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T17:37:33,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741837_1013 (size=414) 2024-11-24T17:37:34,373 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fea1dd1560a14178b1cfa0f9d04f61a, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:37:34,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741838_1014 (size=65) 2024-11-24T17:37:34,786 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:34,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 0fea1dd1560a14178b1cfa0f9d04f61a, disabling compactions & flushes 2024-11-24T17:37:34,787 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:34,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:34,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. after waiting 0 ms 2024-11-24T17:37:34,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:34,788 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:34,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fea1dd1560a14178b1cfa0f9d04f61a: Waiting for close lock at 1732469854787Disabling compacts and flushes for region at 1732469854787Disabling writes for close at 1732469854788 (+1 ms)Writing region close event to WAL at 1732469854788Closed at 1732469854788 2024-11-24T17:37:34,791 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:37:34,792 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732469854791"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469854791"}]},"ts":"1732469854791"} 2024-11-24T17:37:34,797 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:37:34,799 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:37:34,800 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469854799"}]},"ts":"1732469854799"} 2024-11-24T17:37:34,803 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-24T17:37:34,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0fea1dd1560a14178b1cfa0f9d04f61a, ASSIGN}] 2024-11-24T17:37:34,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0fea1dd1560a14178b1cfa0f9d04f61a, ASSIGN 2024-11-24T17:37:34,806 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0fea1dd1560a14178b1cfa0f9d04f61a, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:37:34,957 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0fea1dd1560a14178b1cfa0f9d04f61a, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:34,962 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0fea1dd1560a14178b1cfa0f9d04f61a, ASSIGN because future has completed 2024-11-24T17:37:34,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fea1dd1560a14178b1cfa0f9d04f61a, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:37:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T17:37:35,131 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:35,131 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fea1dd1560a14178b1cfa0f9d04f61a, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:37:35,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:35,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,135 INFO [StoreOpener-0fea1dd1560a14178b1cfa0f9d04f61a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,138 INFO [StoreOpener-0fea1dd1560a14178b1cfa0f9d04f61a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fea1dd1560a14178b1cfa0f9d04f61a columnFamilyName testFamily 2024-11-24T17:37:35,138 DEBUG [StoreOpener-0fea1dd1560a14178b1cfa0f9d04f61a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:35,139 INFO [StoreOpener-0fea1dd1560a14178b1cfa0f9d04f61a-1 {}] regionserver.HStore(327): Store=0fea1dd1560a14178b1cfa0f9d04f61a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:37:35,139 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,140 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,144 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:37:35,149 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 0fea1dd1560a14178b1cfa0f9d04f61a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65895132, jitterRate=-0.018086016178131104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:37:35,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:37:35,150 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0fea1dd1560a14178b1cfa0f9d04f61a: Running coprocessor pre-open hook at 1732469855133Writing region info on filesystem at 1732469855133Initializing all the Stores at 1732469855135 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469855135Cleaning up temporary data from old regions at 1732469855142 (+7 ms)Running coprocessor post-open hooks at 1732469855149 (+7 ms)Region opened successfully at 1732469855150 (+1 ms) 2024-11-24T17:37:35,151 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., pid=9, masterSystemTime=1732469855119 2024-11-24T17:37:35,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:35,155 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:37:35,156 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0fea1dd1560a14178b1cfa0f9d04f61a, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:35,160 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fea1dd1560a14178b1cfa0f9d04f61a, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:37:35,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T17:37:35,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fea1dd1560a14178b1cfa0f9d04f61a, server=a23eb99f71f0,34897,1732469786783 in 199 msec 2024-11-24T17:37:35,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T17:37:35,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0fea1dd1560a14178b1cfa0f9d04f61a, ASSIGN in 363 msec 2024-11-24T17:37:35,172 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:37:35,172 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469855172"}]},"ts":"1732469855172"} 2024-11-24T17:37:35,175 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-24T17:37:35,177 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:37:35,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2270 sec 2024-11-24T17:37:35,949 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:37:35,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-24T17:37:35,977 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T17:37:35,978 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-24T17:37:36,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T17:37:36,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T17:37:36,345 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T17:37:36,345 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T17:37:36,346 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-24T17:37:36,347 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T17:37:36,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-24T17:37:36,348 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-24T17:37:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T17:37:36,996 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-24T17:37:36,996 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-24T17:37:36,997 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:37,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-24T17:37:37,005 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:37,005 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-24T17:37:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:37:37,022 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:37:37,022 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-24T17:37:37,026 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T17:37:37,027 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-24T17:37:43,044 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a23eb99f71f0:34897 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6013ms, rpcTimeout=5999ms 2024-11-24T17:37:43,045 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6013ms, rpcTimeout=5999ms 2024-11-24T17:37:43,045 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 from cache 2024-11-24T17:37:43,054 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5850 (was 5963) 2024-11-24T17:37:43,062 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=5850 2024-11-24T17:37:43,065 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:37:43,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-24T17:37:43,069 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:37:43,070 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:43,070 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-24T17:37:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-24T17:37:43,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:37:43,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741839_1015 (size=412) 2024-11-24T17:37:43,487 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5cb83b0a7cbc6abd6f9b140957151cf9, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:37:43,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741840_1016 (size=63) 2024-11-24T17:37:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-24T17:37:43,902 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:43,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 5cb83b0a7cbc6abd6f9b140957151cf9, disabling compactions & flushes 2024-11-24T17:37:43,903 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:43,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:43,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. after waiting 0 ms 2024-11-24T17:37:43,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:43,903 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:43,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5cb83b0a7cbc6abd6f9b140957151cf9: Waiting for close lock at 1732469863903Disabling compacts and flushes for region at 1732469863903Disabling writes for close at 1732469863903Writing region close event to WAL at 1732469863903Closed at 1732469863903 2024-11-24T17:37:43,906 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:37:43,906 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732469863906"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469863906"}]},"ts":"1732469863906"} 2024-11-24T17:37:43,911 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:37:43,913 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:37:43,913 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469863913"}]},"ts":"1732469863913"} 2024-11-24T17:37:43,916 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-24T17:37:43,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=5cb83b0a7cbc6abd6f9b140957151cf9, ASSIGN}] 2024-11-24T17:37:43,918 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=5cb83b0a7cbc6abd6f9b140957151cf9, ASSIGN 2024-11-24T17:37:43,919 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=5cb83b0a7cbc6abd6f9b140957151cf9, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:37:44,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=5cb83b0a7cbc6abd6f9b140957151cf9, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:44,080 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=5cb83b0a7cbc6abd6f9b140957151cf9, ASSIGN because future has completed 2024-11-24T17:37:44,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cb83b0a7cbc6abd6f9b140957151cf9, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:37:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-24T17:37:44,252 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:44,252 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cb83b0a7cbc6abd6f9b140957151cf9, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:37:44,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:44,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,255 INFO [StoreOpener-5cb83b0a7cbc6abd6f9b140957151cf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,257 INFO [StoreOpener-5cb83b0a7cbc6abd6f9b140957151cf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb83b0a7cbc6abd6f9b140957151cf9 columnFamilyName testFamily 2024-11-24T17:37:44,257 DEBUG [StoreOpener-5cb83b0a7cbc6abd6f9b140957151cf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:44,258 INFO [StoreOpener-5cb83b0a7cbc6abd6f9b140957151cf9-1 {}] regionserver.HStore(327): Store=5cb83b0a7cbc6abd6f9b140957151cf9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:37:44,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,260 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,260 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:37:44,268 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 5cb83b0a7cbc6abd6f9b140957151cf9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61696747, jitterRate=-0.0806468278169632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:37:44,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:37:44,269 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 5cb83b0a7cbc6abd6f9b140957151cf9: Running coprocessor pre-open hook at 1732469864253Writing region info on filesystem at 1732469864253Initializing all the Stores at 1732469864255 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469864255Cleaning up temporary data from old regions at 1732469864261 (+6 ms)Running coprocessor post-open hooks at 1732469864268 (+7 ms)Region opened successfully at 1732469864269 (+1 ms) 2024-11-24T17:37:44,270 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9., pid=12, masterSystemTime=1732469864236 2024-11-24T17:37:44,273 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:44,274 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:37:44,275 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=5cb83b0a7cbc6abd6f9b140957151cf9, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:44,278 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cb83b0a7cbc6abd6f9b140957151cf9, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:37:44,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T17:37:44,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 5cb83b0a7cbc6abd6f9b140957151cf9, server=a23eb99f71f0,34897,1732469786783 in 199 msec 2024-11-24T17:37:44,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T17:37:44,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=5cb83b0a7cbc6abd6f9b140957151cf9, ASSIGN in 368 msec 2024-11-24T17:37:44,289 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:37:44,289 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469864289"}]},"ts":"1732469864289"} 2024-11-24T17:37:44,292 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-24T17:37:44,293 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:37:44,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2280 sec 2024-11-24T17:37:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-24T17:37:46,114 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-24T17:37:46,115 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-24T17:37:46,115 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:46,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-24T17:37:46,119 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:46,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-24T17:37:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:37:46,134 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:37:46,134 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-24T17:37:46,137 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T17:37:46,139 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T17:37:46,141 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T17:37:46,142 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-24T17:37:46,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-24T17:37:46,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T17:37:47,980 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-24T17:37:49,785 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:37:52,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34897 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465963' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T17:37:52,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34897 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:58888 deadline: 1732469878145, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465963' does not exist 2024-11-24T17:37:52,150 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a23eb99f71f0,34897,1732469786783 for closing scanner id = -2479531022807465963 for 5cb83b0a7cbc6abd6f9b140957151cf9 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465963' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:37:52,153 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=111 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=5838 (was 5850) 2024-11-24T17:37:52,163 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=111, ProcessCount=11, AvailableMemoryMB=5837 2024-11-24T17:37:52,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:37:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-24T17:37:52,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:37:52,170 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:52,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-24T17:37:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T17:37:52,172 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:37:52,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741841_1017 (size=423) 2024-11-24T17:37:52,591 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4ad9c9ae020b5952e8b085cac9777211, NAME => 'testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:37:52,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741842_1018 (size=74) 2024-11-24T17:37:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-24T17:37:53,008 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:53,008 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 4ad9c9ae020b5952e8b085cac9777211, disabling compactions & flushes 2024-11-24T17:37:53,008 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,008 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,008 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. after waiting 0 ms 2024-11-24T17:37:53,009 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,009 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,009 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4ad9c9ae020b5952e8b085cac9777211: Waiting for close lock at 1732469873008Disabling compacts and flushes for region at 1732469873008Disabling writes for close at 1732469873009 (+1 ms)Writing region close event to WAL at 1732469873009Closed at 1732469873009 2024-11-24T17:37:53,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:37:53,013 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732469873013"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469873013"}]},"ts":"1732469873013"} 2024-11-24T17:37:53,016 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:37:53,018 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:37:53,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469873018"}]},"ts":"1732469873018"} 2024-11-24T17:37:53,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-24T17:37:53,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4ad9c9ae020b5952e8b085cac9777211, ASSIGN}] 2024-11-24T17:37:53,023 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4ad9c9ae020b5952e8b085cac9777211, ASSIGN 2024-11-24T17:37:53,025 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4ad9c9ae020b5952e8b085cac9777211, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:37:53,176 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4ad9c9ae020b5952e8b085cac9777211, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:53,183 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4ad9c9ae020b5952e8b085cac9777211, ASSIGN because future has completed 2024-11-24T17:37:53,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ad9c9ae020b5952e8b085cac9777211, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:37:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T17:37:53,352 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,353 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ad9c9ae020b5952e8b085cac9777211, NAME => 'testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:37:53,353 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,353 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:53,353 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,353 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,356 INFO [StoreOpener-4ad9c9ae020b5952e8b085cac9777211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,360 INFO [StoreOpener-4ad9c9ae020b5952e8b085cac9777211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ad9c9ae020b5952e8b085cac9777211 columnFamilyName testFamily 2024-11-24T17:37:53,361 DEBUG [StoreOpener-4ad9c9ae020b5952e8b085cac9777211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:53,362 INFO [StoreOpener-4ad9c9ae020b5952e8b085cac9777211-1 {}] regionserver.HStore(327): Store=4ad9c9ae020b5952e8b085cac9777211/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:37:53,362 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,363 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,364 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,364 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,365 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,367 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,371 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:37:53,371 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 4ad9c9ae020b5952e8b085cac9777211; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67760176, jitterRate=0.009705305099487305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:37:53,372 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:37:53,372 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 4ad9c9ae020b5952e8b085cac9777211: Running coprocessor pre-open hook at 1732469873353Writing region info on filesystem at 1732469873353Initializing all the Stores at 1732469873355 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469873355Cleaning up temporary data from old regions at 1732469873365 (+10 ms)Running coprocessor post-open hooks at 1732469873372 (+7 ms)Region opened successfully at 1732469873372 2024-11-24T17:37:53,373 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211., pid=15, masterSystemTime=1732469873340 2024-11-24T17:37:53,376 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,376 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:37:53,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4ad9c9ae020b5952e8b085cac9777211, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:53,380 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ad9c9ae020b5952e8b085cac9777211, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:37:53,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T17:37:53,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ad9c9ae020b5952e8b085cac9777211, server=a23eb99f71f0,34897,1732469786783 in 198 msec 2024-11-24T17:37:53,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T17:37:53,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=4ad9c9ae020b5952e8b085cac9777211, ASSIGN in 364 msec 2024-11-24T17:37:53,390 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:37:53,390 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469873390"}]},"ts":"1732469873390"} 2024-11-24T17:37:53,393 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-24T17:37:53,394 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:37:53,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2280 sec 2024-11-24T17:37:53,982 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-24T17:37:55,059 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T17:37:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T17:37:55,197 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-24T17:37:55,197 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-24T17:37:55,198 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-24T17:37:55,206 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:55,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-24T17:37:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:37:55,221 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:37:55,221 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-24T17:37:55,224 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-24T17:37:55,226 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-24T17:37:55,228 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-24T17:37:55,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2479531022807465959 2024-11-24T17:37:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469878227, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:55,232 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-24T17:37:55,234 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-24T17:37:55,234 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:55,236 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:55,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2479531022807465957 2024-11-24T17:37:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469878236, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:55,241 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:55,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-2479531022807465956 2024-11-24T17:37:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469878241, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:55,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:55,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-2479531022807465955 2024-11-24T17:37:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:58888 deadline: 1732469878246, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T17:37:55,251 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T17:37:55,259 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_581555434_22 at /127.0.0.1:60630 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=102 (was 111), ProcessCount=11 (was 11), AvailableMemoryMB=5790 (was 5837) 2024-11-24T17:37:55,270 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=102, ProcessCount=11, AvailableMemoryMB=5790 2024-11-24T17:37:55,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:37:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-24T17:37:55,277 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:37:55,277 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:55,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-24T17:37:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-24T17:37:55,279 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:37:55,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741843_1019 (size=418) 2024-11-24T17:37:55,697 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 79a8c8dba8d7910f963a849de499f20a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:37:55,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741844_1020 (size=69) 2024-11-24T17:37:56,108 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:56,108 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 79a8c8dba8d7910f963a849de499f20a, disabling compactions & flushes 2024-11-24T17:37:56,109 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,109 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,109 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. after waiting 0 ms 2024-11-24T17:37:56,109 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,109 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,109 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 79a8c8dba8d7910f963a849de499f20a: Waiting for close lock at 1732469876108Disabling compacts and flushes for region at 1732469876108Disabling writes for close at 1732469876109 (+1 ms)Writing region close event to WAL at 1732469876109Closed at 1732469876109 2024-11-24T17:37:56,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:37:56,112 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732469876111"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469876111"}]},"ts":"1732469876111"} 2024-11-24T17:37:56,116 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:37:56,119 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:37:56,119 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469876119"}]},"ts":"1732469876119"} 2024-11-24T17:37:56,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-24T17:37:56,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=79a8c8dba8d7910f963a849de499f20a, ASSIGN}] 2024-11-24T17:37:56,126 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=79a8c8dba8d7910f963a849de499f20a, ASSIGN 2024-11-24T17:37:56,128 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=79a8c8dba8d7910f963a849de499f20a, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:37:56,280 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=79a8c8dba8d7910f963a849de499f20a, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:56,288 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=79a8c8dba8d7910f963a849de499f20a, ASSIGN because future has completed 2024-11-24T17:37:56,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79a8c8dba8d7910f963a849de499f20a, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:37:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-24T17:37:56,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-24T17:37:56,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-24T17:37:56,452 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,453 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 79a8c8dba8d7910f963a849de499f20a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:37:56,453 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,453 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:37:56,453 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,453 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,455 INFO [StoreOpener-79a8c8dba8d7910f963a849de499f20a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,457 INFO [StoreOpener-79a8c8dba8d7910f963a849de499f20a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79a8c8dba8d7910f963a849de499f20a columnFamilyName testFamily 2024-11-24T17:37:56,457 DEBUG [StoreOpener-79a8c8dba8d7910f963a849de499f20a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:37:56,458 INFO [StoreOpener-79a8c8dba8d7910f963a849de499f20a-1 {}] regionserver.HStore(327): Store=79a8c8dba8d7910f963a849de499f20a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:37:56,458 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,459 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,460 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,460 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,461 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,463 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,466 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:37:56,467 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 79a8c8dba8d7910f963a849de499f20a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59914827, jitterRate=-0.10719950497150421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:37:56,467 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:37:56,468 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 79a8c8dba8d7910f963a849de499f20a: Running coprocessor pre-open hook at 1732469876453Writing region info on filesystem at 1732469876453Initializing all the Stores at 1732469876455 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469876455Cleaning up temporary data from old regions at 1732469876461 (+6 ms)Running coprocessor post-open hooks at 1732469876467 (+6 ms)Region opened successfully at 1732469876468 (+1 ms) 2024-11-24T17:37:56,469 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., pid=18, masterSystemTime=1732469876442 2024-11-24T17:37:56,472 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,472 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:37:56,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=79a8c8dba8d7910f963a849de499f20a, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:37:56,475 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79a8c8dba8d7910f963a849de499f20a, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:37:56,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T17:37:56,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 79a8c8dba8d7910f963a849de499f20a, server=a23eb99f71f0,34897,1732469786783 in 189 msec 2024-11-24T17:37:56,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T17:37:56,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=79a8c8dba8d7910f963a849de499f20a, ASSIGN in 357 msec 2024-11-24T17:37:56,485 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:37:56,485 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469876485"}]},"ts":"1732469876485"} 2024-11-24T17:37:56,487 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-24T17:37:56,488 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:37:56,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2150 sec 2024-11-24T17:37:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-24T17:37:58,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-24T17:37:58,317 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-24T17:37:58,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:58,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-24T17:37:58,330 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:37:58,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-24T17:37:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:37:58,345 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:37:58,345 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-24T17:37:58,349 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T17:37:58,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T17:37:58,857 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:37:59,352 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a23eb99f71f0:34897 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-24T17:37:59,353 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-24T17:37:59,353 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 from cache 2024-11-24T17:37:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-24T17:37:59,984 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-24T17:38:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:00,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T17:38:01,352 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a23eb99f71f0:34897 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=978ms, rpcTimeout=976ms 2024-11-24T17:38:01,352 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=978ms, rpcTimeout=976ms 2024-11-24T17:38:01,353 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 from cache 2024-11-24T17:38:01,367 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=93 (was 102), ProcessCount=11 (was 11), AvailableMemoryMB=5781 (was 5790) 2024-11-24T17:38:01,377 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=93, ProcessCount=11, AvailableMemoryMB=5781 2024-11-24T17:38:01,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:38:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-24T17:38:01,384 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:38:01,384 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:01,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-24T17:38:01,385 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:38:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T17:38:01,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741845_1021 (size=416) 2024-11-24T17:38:01,804 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9838185cebea3a0872d5c8915ed6110b, NAME => 'testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:38:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741846_1022 (size=67) 2024-11-24T17:38:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-24T17:38:02,217 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:02,218 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9838185cebea3a0872d5c8915ed6110b, disabling compactions & flushes 2024-11-24T17:38:02,218 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,218 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,219 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. after waiting 0 ms 2024-11-24T17:38:02,219 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,219 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,220 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9838185cebea3a0872d5c8915ed6110b: Waiting for close lock at 1732469882218Disabling compacts and flushes for region at 1732469882218Disabling writes for close at 1732469882219 (+1 ms)Writing region close event to WAL at 1732469882219Closed at 1732469882219 2024-11-24T17:38:02,223 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:38:02,223 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732469882223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469882223"}]},"ts":"1732469882223"} 2024-11-24T17:38:02,226 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:38:02,228 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:38:02,228 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469882228"}]},"ts":"1732469882228"} 2024-11-24T17:38:02,231 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-24T17:38:02,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9838185cebea3a0872d5c8915ed6110b, ASSIGN}] 2024-11-24T17:38:02,233 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9838185cebea3a0872d5c8915ed6110b, ASSIGN 2024-11-24T17:38:02,235 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9838185cebea3a0872d5c8915ed6110b, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:38:02,386 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9838185cebea3a0872d5c8915ed6110b, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:02,394 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9838185cebea3a0872d5c8915ed6110b, ASSIGN because future has completed 2024-11-24T17:38:02,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9838185cebea3a0872d5c8915ed6110b, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:38:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T17:38:02,562 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,562 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 9838185cebea3a0872d5c8915ed6110b, NAME => 'testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:38:02,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:02,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,564 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,566 INFO [StoreOpener-9838185cebea3a0872d5c8915ed6110b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,568 INFO [StoreOpener-9838185cebea3a0872d5c8915ed6110b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9838185cebea3a0872d5c8915ed6110b columnFamilyName testFamily 2024-11-24T17:38:02,568 DEBUG [StoreOpener-9838185cebea3a0872d5c8915ed6110b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:02,569 INFO [StoreOpener-9838185cebea3a0872d5c8915ed6110b-1 {}] regionserver.HStore(327): Store=9838185cebea3a0872d5c8915ed6110b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:38:02,569 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,570 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,573 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,577 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:38:02,577 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 9838185cebea3a0872d5c8915ed6110b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69118656, jitterRate=0.02994823455810547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:38:02,578 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:02,578 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 9838185cebea3a0872d5c8915ed6110b: Running coprocessor pre-open hook at 1732469882564Writing region info on filesystem at 1732469882564Initializing all the Stores at 1732469882566 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469882566Cleaning up temporary data from old regions at 1732469882571 (+5 ms)Running coprocessor post-open hooks at 1732469882578 (+7 ms)Region opened successfully at 1732469882578 2024-11-24T17:38:02,579 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b., pid=21, masterSystemTime=1732469882549 2024-11-24T17:38:02,582 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,582 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:02,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9838185cebea3a0872d5c8915ed6110b, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:02,585 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9838185cebea3a0872d5c8915ed6110b, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:38:02,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T17:38:02,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 9838185cebea3a0872d5c8915ed6110b, server=a23eb99f71f0,34897,1732469786783 in 191 msec 2024-11-24T17:38:02,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T17:38:02,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9838185cebea3a0872d5c8915ed6110b, ASSIGN in 358 msec 2024-11-24T17:38:02,593 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:38:02,593 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469882593"}]},"ts":"1732469882593"} 2024-11-24T17:38:02,595 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-24T17:38:02,597 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:38:02,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2170 sec 2024-11-24T17:38:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T17:38:04,415 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-24T17:38:04,415 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-24T17:38:04,416 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:04,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-24T17:38:04,423 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:04,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-24T17:38:04,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:04,438 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:38:04,439 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-24T17:38:04,442 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T17:38:04,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T17:38:04,445 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T17:38:04,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-24T17:38:05,987 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-24T17:38:06,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-24T17:38:06,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-24T17:38:06,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-24T17:38:06,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T17:38:07,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34897 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465944' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T17:38:07,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34897 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:58888 deadline: 1732469888454, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465944' does not exist 2024-11-24T17:38:07,456 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a23eb99f71f0,34897,1732469786783 for closing scanner id = -2479531022807465944 for 9838185cebea3a0872d5c8915ed6110b of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465944' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:38:07,461 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=86 (was 93), ProcessCount=11 (was 11), AvailableMemoryMB=5773 (was 5781) 2024-11-24T17:38:07,468 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=86, ProcessCount=11, AvailableMemoryMB=5776 2024-11-24T17:38:07,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:38:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-24T17:38:07,473 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:38:07,473 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:07,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-24T17:38:07,474 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:38:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-24T17:38:07,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741847_1023 (size=411) 2024-11-24T17:38:07,890 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2598aa5bbf9a1a0e256d89173660521c, NAME => 'testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:38:07,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741848_1024 (size=62) 2024-11-24T17:38:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-24T17:38:08,069 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:38:08,303 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:08,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 2598aa5bbf9a1a0e256d89173660521c, disabling compactions & flushes 2024-11-24T17:38:08,304 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. after waiting 0 ms 2024-11-24T17:38:08,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,304 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,304 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2598aa5bbf9a1a0e256d89173660521c: Waiting for close lock at 1732469888304Disabling compacts and flushes for region at 1732469888304Disabling writes for close at 1732469888304Writing region close event to WAL at 1732469888304Closed at 1732469888304 2024-11-24T17:38:08,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:38:08,308 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732469888307"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469888307"}]},"ts":"1732469888307"} 2024-11-24T17:38:08,312 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:38:08,314 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:38:08,314 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469888314"}]},"ts":"1732469888314"} 2024-11-24T17:38:08,317 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-24T17:38:08,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=2598aa5bbf9a1a0e256d89173660521c, ASSIGN}] 2024-11-24T17:38:08,319 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=2598aa5bbf9a1a0e256d89173660521c, ASSIGN 2024-11-24T17:38:08,320 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=2598aa5bbf9a1a0e256d89173660521c, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:38:08,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2598aa5bbf9a1a0e256d89173660521c, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:08,479 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=2598aa5bbf9a1a0e256d89173660521c, ASSIGN because future has completed 2024-11-24T17:38:08,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2598aa5bbf9a1a0e256d89173660521c, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:38:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-24T17:38:08,648 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,649 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 2598aa5bbf9a1a0e256d89173660521c, NAME => 'testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:38:08,650 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,650 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:08,650 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,650 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,653 INFO [StoreOpener-2598aa5bbf9a1a0e256d89173660521c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,655 INFO [StoreOpener-2598aa5bbf9a1a0e256d89173660521c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2598aa5bbf9a1a0e256d89173660521c columnFamilyName testFamily 2024-11-24T17:38:08,655 DEBUG [StoreOpener-2598aa5bbf9a1a0e256d89173660521c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:08,656 INFO [StoreOpener-2598aa5bbf9a1a0e256d89173660521c-1 {}] regionserver.HStore(327): Store=2598aa5bbf9a1a0e256d89173660521c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:38:08,656 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,658 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,658 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,659 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,659 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,663 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,667 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:38:08,668 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 2598aa5bbf9a1a0e256d89173660521c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65479981, jitterRate=-0.024272248148918152}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:38:08,668 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:08,669 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 2598aa5bbf9a1a0e256d89173660521c: Running coprocessor pre-open hook at 1732469888650Writing region info on filesystem at 1732469888650Initializing all the Stores at 1732469888652 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469888652Cleaning up temporary data from old regions at 1732469888659 (+7 ms)Running coprocessor post-open hooks at 1732469888668 (+9 ms)Region opened successfully at 1732469888669 (+1 ms) 2024-11-24T17:38:08,670 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c., pid=24, masterSystemTime=1732469888635 2024-11-24T17:38:08,673 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,673 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:08,674 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2598aa5bbf9a1a0e256d89173660521c, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:08,676 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2598aa5bbf9a1a0e256d89173660521c, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:38:08,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T17:38:08,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 2598aa5bbf9a1a0e256d89173660521c, server=a23eb99f71f0,34897,1732469786783 in 197 msec 2024-11-24T17:38:08,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-24T17:38:08,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=2598aa5bbf9a1a0e256d89173660521c, ASSIGN in 363 msec 2024-11-24T17:38:08,684 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:38:08,684 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469888684"}]},"ts":"1732469888684"} 2024-11-24T17:38:08,687 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-24T17:38:08,689 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:38:08,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2190 sec 2024-11-24T17:38:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-24T17:38:10,494 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-24T17:38:10,494 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-24T17:38:10,495 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:10,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-24T17:38:10,500 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:10,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-24T17:38:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:10,516 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:38:10,516 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-24T17:38:10,520 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T17:38:10,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T17:38:10,524 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T17:38:10,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-24T17:38:10,720 INFO [master/a23eb99f71f0:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T17:38:10,721 INFO [master/a23eb99f71f0:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T17:38:11,988 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-24T17:38:13,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34897 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465940' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T17:38:13,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34897 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:58888 deadline: 1732469894534, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465940' does not exist 2024-11-24T17:38:13,543 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a23eb99f71f0,34897,1732469786783 for closing scanner id = -2479531022807465940 for 2598aa5bbf9a1a0e256d89173660521c of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465940' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:38:13,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=80 (was 86), ProcessCount=11 (was 11), AvailableMemoryMB=5769 (was 5776) 2024-11-24T17:38:13,562 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=80, ProcessCount=11, AvailableMemoryMB=5769 2024-11-24T17:38:13,566 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:38:13,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-24T17:38:13,570 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:38:13,570 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:13,570 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-24T17:38:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T17:38:13,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:38:13,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741849_1025 (size=419) 2024-11-24T17:38:13,988 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 844f29b4217aafa54fc11bf6d2a46339, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:38:13,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741850_1026 (size=70) 2024-11-24T17:38:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-24T17:38:14,154 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:38:14,403 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:14,403 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 844f29b4217aafa54fc11bf6d2a46339, disabling compactions & flushes 2024-11-24T17:38:14,403 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. after waiting 0 ms 2024-11-24T17:38:14,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,404 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 844f29b4217aafa54fc11bf6d2a46339: Waiting for close lock at 1732469894403Disabling compacts and flushes for region at 1732469894403Disabling writes for close at 1732469894404 (+1 ms)Writing region close event to WAL at 1732469894404Closed at 1732469894404 2024-11-24T17:38:14,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:38:14,409 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732469894409"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469894409"}]},"ts":"1732469894409"} 2024-11-24T17:38:14,413 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:38:14,415 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:38:14,415 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469894415"}]},"ts":"1732469894415"} 2024-11-24T17:38:14,418 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-24T17:38:14,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=844f29b4217aafa54fc11bf6d2a46339, ASSIGN}] 2024-11-24T17:38:14,420 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=844f29b4217aafa54fc11bf6d2a46339, ASSIGN 2024-11-24T17:38:14,421 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=844f29b4217aafa54fc11bf6d2a46339, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:38:14,573 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=844f29b4217aafa54fc11bf6d2a46339, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:14,580 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=844f29b4217aafa54fc11bf6d2a46339, ASSIGN because future has completed 2024-11-24T17:38:14,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 844f29b4217aafa54fc11bf6d2a46339, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:38:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T17:38:14,742 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 844f29b4217aafa54fc11bf6d2a46339, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:38:14,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:14,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,747 INFO [StoreOpener-844f29b4217aafa54fc11bf6d2a46339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,750 INFO [StoreOpener-844f29b4217aafa54fc11bf6d2a46339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 844f29b4217aafa54fc11bf6d2a46339 columnFamilyName testFamily 2024-11-24T17:38:14,750 DEBUG [StoreOpener-844f29b4217aafa54fc11bf6d2a46339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:14,751 INFO [StoreOpener-844f29b4217aafa54fc11bf6d2a46339-1 {}] regionserver.HStore(327): Store=844f29b4217aafa54fc11bf6d2a46339/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:38:14,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:38:14,759 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 844f29b4217aafa54fc11bf6d2a46339; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71249455, jitterRate=0.06169961392879486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:38:14,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:14,760 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 844f29b4217aafa54fc11bf6d2a46339: Running coprocessor pre-open hook at 1732469894744Writing region info on filesystem at 1732469894744Initializing all the Stores at 1732469894746 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469894746Cleaning up temporary data from old regions at 1732469894754 (+8 ms)Running coprocessor post-open hooks at 1732469894759 (+5 ms)Region opened successfully at 1732469894760 (+1 ms) 2024-11-24T17:38:14,761 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., pid=27, masterSystemTime=1732469894734 2024-11-24T17:38:14,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:14,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=844f29b4217aafa54fc11bf6d2a46339, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:14,766 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 844f29b4217aafa54fc11bf6d2a46339, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:38:14,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T17:38:14,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 844f29b4217aafa54fc11bf6d2a46339, server=a23eb99f71f0,34897,1732469786783 in 186 msec 2024-11-24T17:38:14,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T17:38:14,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=844f29b4217aafa54fc11bf6d2a46339, ASSIGN in 352 msec 2024-11-24T17:38:14,773 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:38:14,773 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469894773"}]},"ts":"1732469894773"} 2024-11-24T17:38:14,776 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-24T17:38:14,777 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:38:14,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2110 sec 2024-11-24T17:38:16,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-24T17:38:16,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-24T17:38:16,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-24T17:38:16,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T17:38:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T17:38:16,595 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-24T17:38:16,595 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-24T17:38:16,595 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:16,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-24T17:38:16,600 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:16,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-24T17:38:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:16,613 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:38:16,613 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-24T17:38:16,616 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T17:38:16,618 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-24T17:38:17,990 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-24T17:38:20,246 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:38:22,622 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a23eb99f71f0:34897 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-11-24T17:38:22,622 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-11-24T17:38:22,623 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 from cache 2024-11-24T17:38:22,639 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=74 (was 80), ProcessCount=11 (was 11), AvailableMemoryMB=5729 (was 5769) 2024-11-24T17:38:22,651 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=74, ProcessCount=11, AvailableMemoryMB=5729 2024-11-24T17:38:22,654 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:38:22,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-24T17:38:22,659 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:38:22,659 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:22,659 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-24T17:38:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-24T17:38:22,661 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:38:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741851_1027 (size=407) 2024-11-24T17:38:23,078 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f631bc47193ffa8011bf43eae75ce5b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:38:23,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741852_1028 (size=58) 2024-11-24T17:38:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-24T17:38:23,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:23,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 9f631bc47193ffa8011bf43eae75ce5b, disabling compactions & flushes 2024-11-24T17:38:23,490 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. after waiting 0 ms 2024-11-24T17:38:23,491 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,491 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,491 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f631bc47193ffa8011bf43eae75ce5b: Waiting for close lock at 1732469903490Disabling compacts and flushes for region at 1732469903490Disabling writes for close at 1732469903490Writing region close event to WAL at 1732469903491 (+1 ms)Closed at 1732469903491 2024-11-24T17:38:23,494 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:38:23,494 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732469903494"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469903494"}]},"ts":"1732469903494"} 2024-11-24T17:38:23,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:38:23,501 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:38:23,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469903501"}]},"ts":"1732469903501"} 2024-11-24T17:38:23,503 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-24T17:38:23,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=9f631bc47193ffa8011bf43eae75ce5b, ASSIGN}] 2024-11-24T17:38:23,505 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=9f631bc47193ffa8011bf43eae75ce5b, ASSIGN 2024-11-24T17:38:23,506 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=9f631bc47193ffa8011bf43eae75ce5b, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:38:23,657 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9f631bc47193ffa8011bf43eae75ce5b, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:23,663 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=9f631bc47193ffa8011bf43eae75ce5b, ASSIGN because future has completed 2024-11-24T17:38:23,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f631bc47193ffa8011bf43eae75ce5b, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:38:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-24T17:38:23,827 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,828 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f631bc47193ffa8011bf43eae75ce5b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:38:23,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:23,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,833 INFO [StoreOpener-9f631bc47193ffa8011bf43eae75ce5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,835 INFO [StoreOpener-9f631bc47193ffa8011bf43eae75ce5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f631bc47193ffa8011bf43eae75ce5b columnFamilyName testFamily 2024-11-24T17:38:23,835 DEBUG [StoreOpener-9f631bc47193ffa8011bf43eae75ce5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:23,836 INFO [StoreOpener-9f631bc47193ffa8011bf43eae75ce5b-1 {}] regionserver.HStore(327): Store=9f631bc47193ffa8011bf43eae75ce5b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:38:23,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:38:23,845 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 9f631bc47193ffa8011bf43eae75ce5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67440081, jitterRate=0.004935517907142639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:38:23,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:23,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 9f631bc47193ffa8011bf43eae75ce5b: Running coprocessor pre-open hook at 1732469903830Writing region info on filesystem at 1732469903830Initializing all the Stores at 1732469903832 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469903832Cleaning up temporary data from old regions at 1732469903838 (+6 ms)Running coprocessor post-open hooks at 1732469903845 (+7 ms)Region opened successfully at 1732469903846 (+1 ms) 2024-11-24T17:38:23,847 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b., pid=30, masterSystemTime=1732469903817 2024-11-24T17:38:23,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:23,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9f631bc47193ffa8011bf43eae75ce5b, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:23,853 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f631bc47193ffa8011bf43eae75ce5b, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:38:23,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T17:38:23,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f631bc47193ffa8011bf43eae75ce5b, server=a23eb99f71f0,34897,1732469786783 in 190 msec 2024-11-24T17:38:23,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-24T17:38:23,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=9f631bc47193ffa8011bf43eae75ce5b, ASSIGN in 354 msec 2024-11-24T17:38:23,861 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:38:23,861 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469903861"}]},"ts":"1732469903861"} 2024-11-24T17:38:23,863 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-24T17:38:23,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:38:23,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2090 sec 2024-11-24T17:38:23,993 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-24T17:38:25,059 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T17:38:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-24T17:38:25,686 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-24T17:38:25,687 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-24T17:38:25,687 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:25,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-24T17:38:25,697 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:25,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-24T17:38:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:25,710 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:38:25,711 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-24T17:38:25,713 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T17:38:25,715 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T17:38:25,717 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T17:38:25,717 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-24T17:38:26,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-24T17:38:26,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-24T17:38:29,334 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:38:30,628 DEBUG [master/a23eb99f71f0:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:38:31,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34897 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465932' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T17:38:31,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34897 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:58888 deadline: 1732469917724, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465932' does not exist 2024-11-24T17:38:31,732 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a23eb99f71f0,34897,1732469786783 for closing scanner id = -2479531022807465932 for 9f631bc47193ffa8011bf43eae75ce5b of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2479531022807465932' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:38:31,737 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=84 (was 74) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5720 (was 5729) 2024-11-24T17:38:31,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=84, ProcessCount=11, AvailableMemoryMB=5720 2024-11-24T17:38:31,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:38:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-24T17:38:31,749 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:38:31,749 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:31,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-24T17:38:31,750 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:38:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T17:38:31,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741853_1029 (size=423) 2024-11-24T17:38:32,164 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb0f6b61ef86a4c67176af6eaeb3cd61, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e 2024-11-24T17:38:32,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741854_1030 (size=74) 2024-11-24T17:38:32,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-24T17:38:32,578 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:32,579 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing fb0f6b61ef86a4c67176af6eaeb3cd61, disabling compactions & flushes 2024-11-24T17:38:32,579 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,579 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,579 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. after waiting 0 ms 2024-11-24T17:38:32,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,580 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb0f6b61ef86a4c67176af6eaeb3cd61: Waiting for close lock at 1732469912579Disabling compacts and flushes for region at 1732469912579Disabling writes for close at 1732469912580 (+1 ms)Writing region close event to WAL at 1732469912580Closed at 1732469912580 2024-11-24T17:38:32,584 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:38:32,584 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732469912584"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732469912584"}]},"ts":"1732469912584"} 2024-11-24T17:38:32,587 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T17:38:32,589 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:38:32,589 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469912589"}]},"ts":"1732469912589"} 2024-11-24T17:38:32,592 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-24T17:38:32,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fb0f6b61ef86a4c67176af6eaeb3cd61, ASSIGN}] 2024-11-24T17:38:32,594 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fb0f6b61ef86a4c67176af6eaeb3cd61, ASSIGN 2024-11-24T17:38:32,595 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fb0f6b61ef86a4c67176af6eaeb3cd61, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34897,1732469786783; forceNewPlan=false, retain=false 2024-11-24T17:38:32,746 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=fb0f6b61ef86a4c67176af6eaeb3cd61, regionState=OPENING, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:32,750 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fb0f6b61ef86a4c67176af6eaeb3cd61, ASSIGN because future has completed 2024-11-24T17:38:32,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb0f6b61ef86a4c67176af6eaeb3cd61, server=a23eb99f71f0,34897,1732469786783}] 2024-11-24T17:38:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T17:38:32,914 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,914 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => fb0f6b61ef86a4c67176af6eaeb3cd61, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:38:32,915 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,915 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:38:32,915 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,915 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,919 INFO [StoreOpener-fb0f6b61ef86a4c67176af6eaeb3cd61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,922 INFO [StoreOpener-fb0f6b61ef86a4c67176af6eaeb3cd61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb0f6b61ef86a4c67176af6eaeb3cd61 columnFamilyName testFamily 2024-11-24T17:38:32,922 DEBUG [StoreOpener-fb0f6b61ef86a4c67176af6eaeb3cd61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:38:32,923 INFO [StoreOpener-fb0f6b61ef86a4c67176af6eaeb3cd61-1 {}] regionserver.HStore(327): Store=fb0f6b61ef86a4c67176af6eaeb3cd61/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:38:32,923 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,925 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,925 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,926 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,926 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,929 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,932 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:38:32,933 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened fb0f6b61ef86a4c67176af6eaeb3cd61; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62335267, jitterRate=-0.07113213837146759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:38:32,933 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:32,934 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for fb0f6b61ef86a4c67176af6eaeb3cd61: Running coprocessor pre-open hook at 1732469912916Writing region info on filesystem at 1732469912916Initializing all the Stores at 1732469912918 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732469912918Cleaning up temporary data from old regions at 1732469912926 (+8 ms)Running coprocessor post-open hooks at 1732469912933 (+7 ms)Region opened successfully at 1732469912934 (+1 ms) 2024-11-24T17:38:32,935 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., pid=33, masterSystemTime=1732469912904 2024-11-24T17:38:32,938 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,938 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:32,939 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=fb0f6b61ef86a4c67176af6eaeb3cd61, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:32,941 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb0f6b61ef86a4c67176af6eaeb3cd61, server=a23eb99f71f0,34897,1732469786783 because future has completed 2024-11-24T17:38:32,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-24T17:38:32,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure fb0f6b61ef86a4c67176af6eaeb3cd61, server=a23eb99f71f0,34897,1732469786783 in 192 msec 2024-11-24T17:38:32,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T17:38:32,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fb0f6b61ef86a4c67176af6eaeb3cd61, ASSIGN in 353 msec 2024-11-24T17:38:32,948 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:38:32,948 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732469912948"}]},"ts":"1732469912948"} 2024-11-24T17:38:32,950 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-24T17:38:32,951 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:38:32,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2040 sec 2024-11-24T17:38:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T17:38:34,777 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-24T17:38:34,777 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-24T17:38:34,778 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:34,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-24T17:38:34,787 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:38:34,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-24T17:38:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:34,803 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T17:38:34,803 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-24T17:38:34,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T17:38:34,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T17:38:35,812 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a23eb99f71f0:34897 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-24T17:38:35,812 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-24T17:38:35,813 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 from cache 2024-11-24T17:38:35,997 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-24T17:38:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-24T17:38:36,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-24T17:38:36,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T17:38:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2] 2024-11-24T17:38:36,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34897 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T17:38:37,812 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a23eb99f71f0:34897 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=979ms, rpcTimeout=974ms 2024-11-24T17:38:37,812 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=979ms, rpcTimeout=974ms 2024-11-24T17:38:37,812 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., hostname=a23eb99f71f0,34897,1732469786783, seqNum=2 from cache 2024-11-24T17:38:37,824 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=77 (was 84), ProcessCount=11 (was 11), AvailableMemoryMB=5714 (was 5720) 2024-11-24T17:38:37,825 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T17:38:37,825 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:38:37,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:37,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:37,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T17:38:37,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T17:38:37,832 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T17:38:37,832 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:38:37,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:37,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:37,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T17:38:37,832 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T17:38:37,832 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-11-24T17:38:37,833 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a23eb99f71f0,33353,1732469785982 2024-11-24T17:38:37,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T17:38:37,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T17:38:37,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:38:37,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:38:37,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:38:37,907 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T17:38:37,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:38:37,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:38:37,907 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:38:37,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:37,909 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a23eb99f71f0,34897,1732469786783' ***** 2024-11-24T17:38:37,909 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T17:38:37,910 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T17:38:37,911 INFO [RS:0;a23eb99f71f0:34897 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T17:38:37,911 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T17:38:37,911 INFO [RS:0;a23eb99f71f0:34897 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T17:38:37,911 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 0fea1dd1560a14178b1cfa0f9d04f61a 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 33bcf1d6c44cbf423a9903914d6f35a5 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 9f631bc47193ffa8011bf43eae75ce5b 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 79a8c8dba8d7910f963a849de499f20a 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 5cb83b0a7cbc6abd6f9b140957151cf9 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ad9c9ae020b5952e8b085cac9777211 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:37,912 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0fea1dd1560a14178b1cfa0f9d04f61a, disabling compactions & flushes 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:37,912 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(959): stopping server a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:37,912 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T17:38:37,912 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. after waiting 0 ms 2024-11-24T17:38:37,912 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:38:37,912 INFO [RS:0;a23eb99f71f0:34897 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a23eb99f71f0:34897. 2024-11-24T17:38:37,912 DEBUG [RS:0;a23eb99f71f0:34897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:38:37,913 DEBUG [RS:0;a23eb99f71f0:34897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:37,913 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T17:38:37,913 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T17:38:37,913 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T17:38:37,913 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T17:38:37,913 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-24T17:38:37,913 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T17:38:37,913 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T17:38:37,913 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T17:38:37,914 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T17:38:37,914 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 0fea1dd1560a14178b1cfa0f9d04f61a, 1588230740, 2598aa5bbf9a1a0e256d89173660521c, 33bcf1d6c44cbf423a9903914d6f35a5, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, 9f631bc47193ffa8011bf43eae75ce5b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:37,914 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T17:38:37,916 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0fea1dd1560a14178b1cfa0f9d04f61a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:37,916 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-24T17:38:37,917 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T17:38:37,917 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T17:38:37,967 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/.tmp/testFamily/0d7a8371673847818753601f1d3f20e7 is 61, key is row-0/testFamily:testQualifier/1732469857010/Put/seqid=0 2024-11-24T17:38:37,973 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/info/4816ff04c93e40d4976806a5f2ddd92c is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5./info:regioninfo/1732469852039/Put/seqid=0 2024-11-24T17:38:37,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741855_1031 (size=5049) 2024-11-24T17:38:37,977 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/.tmp/testFamily/0d7a8371673847818753601f1d3f20e7 2024-11-24T17:38:37,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741856_1032 (size=20564) 2024-11-24T17:38:37,979 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/info/4816ff04c93e40d4976806a5f2ddd92c 2024-11-24T17:38:38,019 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/.tmp/testFamily/0d7a8371673847818753601f1d3f20e7 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/testFamily/0d7a8371673847818753601f1d3f20e7 2024-11-24T17:38:38,030 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/testFamily/0d7a8371673847818753601f1d3f20e7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:38,036 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/ns/c0c3158456db464887ab1f8e252faf0a is 43, key is default/ns:d/1732469850577/Put/seqid=0 2024-11-24T17:38:38,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741857_1033 (size=5153) 2024-11-24T17:38:38,041 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0fea1dd1560a14178b1cfa0f9d04f61a in 122ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:38,048 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScanner/0fea1dd1560a14178b1cfa0f9d04f61a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:38,051 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:38:38,051 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0fea1dd1560a14178b1cfa0f9d04f61a: Waiting for close lock at 1732469917912Running coprocessor pre-close hooks at 1732469917912Disabling compacts and flushes for region at 1732469917912Disabling writes for close at 1732469917912Obtaining lock to block concurrent updates at 1732469917916 (+4 ms)Preparing flush snapshotting stores in 0fea1dd1560a14178b1cfa0f9d04f61a at 1732469917916Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469917922 (+6 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. at 1732469917924 (+2 ms)Flushing 0fea1dd1560a14178b1cfa0f9d04f61a/testFamily: creating writer at 1732469917925 (+1 ms)Flushing 0fea1dd1560a14178b1cfa0f9d04f61a/testFamily: appending metadata at 1732469917963 (+38 ms)Flushing 0fea1dd1560a14178b1cfa0f9d04f61a/testFamily: closing flushed file at 1732469917965 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47aefb84: reopening flushed file at 1732469918017 (+52 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0fea1dd1560a14178b1cfa0f9d04f61a in 122ms, sequenceid=8, compaction requested=false at 1732469918041 (+24 ms)Writing region close event to WAL at 1732469918043 (+2 ms)Running coprocessor post-close hooks at 1732469918049 (+6 ms)Closed at 1732469918050 (+1 ms) 2024-11-24T17:38:38,051 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732469853950.0fea1dd1560a14178b1cfa0f9d04f61a. 2024-11-24T17:38:38,051 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 33bcf1d6c44cbf423a9903914d6f35a5, disabling compactions & flushes 2024-11-24T17:38:38,051 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:38:38,052 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:38:38,052 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. after waiting 0 ms 2024-11-24T17:38:38,052 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:38:38,052 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 33bcf1d6c44cbf423a9903914d6f35a5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:38,069 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/.tmp/testFamily/1c487c9f14584edebcff50b951e2cfdb is 61, key is row-0/testFamily:testQualifier/1732469853880/Put/seqid=0 2024-11-24T17:38:38,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741858_1034 (size=5049) 2024-11-24T17:38:38,114 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2598aa5bbf9a1a0e256d89173660521c, 33bcf1d6c44cbf423a9903914d6f35a5, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, 9f631bc47193ffa8011bf43eae75ce5b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:38,314 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2598aa5bbf9a1a0e256d89173660521c, 33bcf1d6c44cbf423a9903914d6f35a5, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, 9f631bc47193ffa8011bf43eae75ce5b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:38,335 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T17:38:38,417 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:38:38,442 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/ns/c0c3158456db464887ab1f8e252faf0a 2024-11-24T17:38:38,475 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/table/a8e4af0ed27e4d6c8764045d94a49b32 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732469852055/Put/seqid=0 2024-11-24T17:38:38,476 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/.tmp/testFamily/1c487c9f14584edebcff50b951e2cfdb 2024-11-24T17:38:38,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741859_1035 (size=6699) 2024-11-24T17:38:38,497 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/.tmp/testFamily/1c487c9f14584edebcff50b951e2cfdb as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/testFamily/1c487c9f14584edebcff50b951e2cfdb 2024-11-24T17:38:38,508 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/testFamily/1c487c9f14584edebcff50b951e2cfdb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:38,509 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 33bcf1d6c44cbf423a9903914d6f35a5 in 457ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:38,515 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2598aa5bbf9a1a0e256d89173660521c, 33bcf1d6c44cbf423a9903914d6f35a5, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, 9f631bc47193ffa8011bf43eae75ce5b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:38,515 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/33bcf1d6c44cbf423a9903914d6f35a5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:38,517 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:38:38,517 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 33bcf1d6c44cbf423a9903914d6f35a5: Waiting for close lock at 1732469918051Running coprocessor pre-close hooks at 1732469918051Disabling compacts and flushes for region at 1732469918051Disabling writes for close at 1732469918052 (+1 ms)Obtaining lock to block concurrent updates at 1732469918052Preparing flush snapshotting stores in 33bcf1d6c44cbf423a9903914d6f35a5 at 1732469918052Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469918052Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. at 1732469918053 (+1 ms)Flushing 33bcf1d6c44cbf423a9903914d6f35a5/testFamily: creating writer at 1732469918053Flushing 33bcf1d6c44cbf423a9903914d6f35a5/testFamily: appending metadata at 1732469918069 (+16 ms)Flushing 33bcf1d6c44cbf423a9903914d6f35a5/testFamily: closing flushed file at 1732469918069Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f3aac3: reopening flushed file at 1732469918495 (+426 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 33bcf1d6c44cbf423a9903914d6f35a5 in 457ms, sequenceid=8, compaction requested=false at 1732469918509 (+14 ms)Writing region close event to WAL at 1732469918510 (+1 ms)Running coprocessor post-close hooks at 1732469918516 (+6 ms)Closed at 1732469918516 2024-11-24T17:38:38,517 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732469850788.33bcf1d6c44cbf423a9903914d6f35a5. 2024-11-24T17:38:38,517 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9f631bc47193ffa8011bf43eae75ce5b, disabling compactions & flushes 2024-11-24T17:38:38,517 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:38,517 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:38,517 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. after waiting 0 ms 2024-11-24T17:38:38,517 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:38,518 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9f631bc47193ffa8011bf43eae75ce5b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:38,541 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/.tmp/testFamily/c05483d9af034825904af26171953522 is 61, key is row-0/testFamily:testQualifier/1732469905703/Put/seqid=0 2024-11-24T17:38:38,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741860_1036 (size=5049) 2024-11-24T17:38:38,715 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, 9f631bc47193ffa8011bf43eae75ce5b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:38,890 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/table/a8e4af0ed27e4d6c8764045d94a49b32 2024-11-24T17:38:38,904 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/info/4816ff04c93e40d4976806a5f2ddd92c as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/info/4816ff04c93e40d4976806a5f2ddd92c 2024-11-24T17:38:38,914 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/info/4816ff04c93e40d4976806a5f2ddd92c, entries=100, sequenceid=56, filesize=20.1 K 2024-11-24T17:38:38,915 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-24T17:38:38,915 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1325): Online Regions={9f631bc47193ffa8011bf43eae75ce5b=hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b., fb0f6b61ef86a4c67176af6eaeb3cd61=testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., 79a8c8dba8d7910f963a849de499f20a=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., 5cb83b0a7cbc6abd6f9b140957151cf9=hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9., 1588230740=hbase:meta,,1.1588230740, 4ad9c9ae020b5952e8b085cac9777211=testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211., 9838185cebea3a0872d5c8915ed6110b=testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b., 844f29b4217aafa54fc11bf6d2a46339=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., 2598aa5bbf9a1a0e256d89173660521c=testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.} 2024-11-24T17:38:38,915 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, 9f631bc47193ffa8011bf43eae75ce5b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:38,916 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/ns/c0c3158456db464887ab1f8e252faf0a as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/ns/c0c3158456db464887ab1f8e252faf0a 2024-11-24T17:38:38,924 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/ns/c0c3158456db464887ab1f8e252faf0a, entries=2, sequenceid=56, filesize=5.0 K 2024-11-24T17:38:38,926 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/.tmp/table/a8e4af0ed27e4d6c8764045d94a49b32 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/table/a8e4af0ed27e4d6c8764045d94a49b32 2024-11-24T17:38:38,934 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/table/a8e4af0ed27e4d6c8764045d94a49b32, entries=20, sequenceid=56, filesize=6.5 K 2024-11-24T17:38:38,935 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1021ms, sequenceid=56, compaction requested=false 2024-11-24T17:38:38,940 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-24T17:38:38,941 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T17:38:38,941 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T17:38:38,942 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732469917913Running coprocessor pre-close hooks at 1732469917913Disabling compacts and flushes for region at 1732469917913Disabling writes for close at 1732469917914 (+1 ms)Obtaining lock to block concurrent updates at 1732469917916 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732469917916Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732469917922 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732469917924 (+2 ms)Flushing 1588230740/info: creating writer at 1732469917925 (+1 ms)Flushing 1588230740/info: appending metadata at 1732469917968 (+43 ms)Flushing 1588230740/info: closing flushed file at 1732469917968Flushing 1588230740/ns: creating writer at 1732469918017 (+49 ms)Flushing 1588230740/ns: appending metadata at 1732469918035 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732469918036 (+1 ms)Flushing 1588230740/table: creating writer at 1732469918452 (+416 ms)Flushing 1588230740/table: appending metadata at 1732469918474 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732469918474Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35487254: reopening flushed file at 1732469918903 (+429 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eabbd2d: reopening flushed file at 1732469918914 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a37a27: reopening flushed file at 1732469918925 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1021ms, sequenceid=56, compaction requested=false at 1732469918935 (+10 ms)Writing region close event to WAL at 1732469918936 (+1 ms)Running coprocessor post-close hooks at 1732469918941 (+5 ms)Closed at 1732469918941 2024-11-24T17:38:38,942 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T17:38:38,948 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/.tmp/testFamily/c05483d9af034825904af26171953522 2024-11-24T17:38:38,958 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/.tmp/testFamily/c05483d9af034825904af26171953522 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/testFamily/c05483d9af034825904af26171953522 2024-11-24T17:38:38,970 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/testFamily/c05483d9af034825904af26171953522, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:38,971 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9f631bc47193ffa8011bf43eae75ce5b in 454ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:38,977 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNext/9f631bc47193ffa8011bf43eae75ce5b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:38,978 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:38,978 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9f631bc47193ffa8011bf43eae75ce5b: Waiting for close lock at 1732469918517Running coprocessor pre-close hooks at 1732469918517Disabling compacts and flushes for region at 1732469918517Disabling writes for close at 1732469918517Obtaining lock to block concurrent updates at 1732469918518 (+1 ms)Preparing flush snapshotting stores in 9f631bc47193ffa8011bf43eae75ce5b at 1732469918518Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469918518Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. at 1732469918519 (+1 ms)Flushing 9f631bc47193ffa8011bf43eae75ce5b/testFamily: creating writer at 1732469918519Flushing 9f631bc47193ffa8011bf43eae75ce5b/testFamily: appending metadata at 1732469918541 (+22 ms)Flushing 9f631bc47193ffa8011bf43eae75ce5b/testFamily: closing flushed file at 1732469918541Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f8e807: reopening flushed file at 1732469918956 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9f631bc47193ffa8011bf43eae75ce5b in 454ms, sequenceid=8, compaction requested=false at 1732469918971 (+15 ms)Writing region close event to WAL at 1732469918972 (+1 ms)Running coprocessor post-close hooks at 1732469918978 (+6 ms)Closed at 1732469918978 2024-11-24T17:38:38,978 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732469902654.9f631bc47193ffa8011bf43eae75ce5b. 2024-11-24T17:38:38,979 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb0f6b61ef86a4c67176af6eaeb3cd61, disabling compactions & flushes 2024-11-24T17:38:38,979 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:38,979 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:38,979 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. after waiting 0 ms 2024-11-24T17:38:38,979 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:38,979 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fb0f6b61ef86a4c67176af6eaeb3cd61 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:38,995 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/.tmp/testFamily/28fe58d48cde449f84090dc6a0dec506 is 61, key is row-0/testFamily:testQualifier/1732469914792/Put/seqid=0 2024-11-24T17:38:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741861_1037 (size=5049) 2024-11-24T17:38:39,116 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:39,316 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b, fb0f6b61ef86a4c67176af6eaeb3cd61 2024-11-24T17:38:39,402 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/.tmp/testFamily/28fe58d48cde449f84090dc6a0dec506 2024-11-24T17:38:39,459 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/.tmp/testFamily/28fe58d48cde449f84090dc6a0dec506 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/testFamily/28fe58d48cde449f84090dc6a0dec506 2024-11-24T17:38:39,472 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/testFamily/28fe58d48cde449f84090dc6a0dec506, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:39,473 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fb0f6b61ef86a4c67176af6eaeb3cd61 in 494ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:39,478 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScannerAsync/fb0f6b61ef86a4c67176af6eaeb3cd61/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:39,479 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:39,479 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb0f6b61ef86a4c67176af6eaeb3cd61: Waiting for close lock at 1732469918979Running coprocessor pre-close hooks at 1732469918979Disabling compacts and flushes for region at 1732469918979Disabling writes for close at 1732469918979Obtaining lock to block concurrent updates at 1732469918979Preparing flush snapshotting stores in fb0f6b61ef86a4c67176af6eaeb3cd61 at 1732469918979Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469918979Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. at 1732469918980 (+1 ms)Flushing fb0f6b61ef86a4c67176af6eaeb3cd61/testFamily: creating writer at 1732469918980Flushing fb0f6b61ef86a4c67176af6eaeb3cd61/testFamily: appending metadata at 1732469918995 (+15 ms)Flushing fb0f6b61ef86a4c67176af6eaeb3cd61/testFamily: closing flushed file at 1732469918995Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f895dab: reopening flushed file at 1732469919457 (+462 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fb0f6b61ef86a4c67176af6eaeb3cd61 in 494ms, sequenceid=8, compaction requested=false at 1732469919473 (+16 ms)Writing region close event to WAL at 1732469919474 (+1 ms)Running coprocessor post-close hooks at 1732469919479 (+5 ms)Closed at 1732469919479 2024-11-24T17:38:39,479 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732469911746.fb0f6b61ef86a4c67176af6eaeb3cd61. 2024-11-24T17:38:39,479 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79a8c8dba8d7910f963a849de499f20a, disabling compactions & flushes 2024-11-24T17:38:39,479 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:38:39,480 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:38:39,480 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. after waiting 0 ms 2024-11-24T17:38:39,480 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:38:39,480 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 79a8c8dba8d7910f963a849de499f20a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:39,495 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/.tmp/testFamily/e035bfecc7604594b50d8cbeaa436c12 is 61, key is row-0/testFamily:testQualifier/1732469878335/Put/seqid=0 2024-11-24T17:38:39,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741862_1038 (size=5049) 2024-11-24T17:38:39,517 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:39,717 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:39,903 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/.tmp/testFamily/e035bfecc7604594b50d8cbeaa436c12 2024-11-24T17:38:39,917 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-24T17:38:39,918 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1325): Online Regions={79a8c8dba8d7910f963a849de499f20a=testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., 5cb83b0a7cbc6abd6f9b140957151cf9=hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9., 4ad9c9ae020b5952e8b085cac9777211=testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211., 9838185cebea3a0872d5c8915ed6110b=testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b., 844f29b4217aafa54fc11bf6d2a46339=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., 2598aa5bbf9a1a0e256d89173660521c=testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.} 2024-11-24T17:38:39,918 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 79a8c8dba8d7910f963a849de499f20a, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:39,918 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/.tmp/testFamily/e035bfecc7604594b50d8cbeaa436c12 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/testFamily/e035bfecc7604594b50d8cbeaa436c12 2024-11-24T17:38:39,927 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/testFamily/e035bfecc7604594b50d8cbeaa436c12, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:39,928 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 79a8c8dba8d7910f963a849de499f20a in 448ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:39,933 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnOpenScanner/79a8c8dba8d7910f963a849de499f20a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:39,934 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:38:39,934 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79a8c8dba8d7910f963a849de499f20a: Waiting for close lock at 1732469919479Running coprocessor pre-close hooks at 1732469919479Disabling compacts and flushes for region at 1732469919479Disabling writes for close at 1732469919480 (+1 ms)Obtaining lock to block concurrent updates at 1732469919480Preparing flush snapshotting stores in 79a8c8dba8d7910f963a849de499f20a at 1732469919480Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469919480Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. at 1732469919481 (+1 ms)Flushing 79a8c8dba8d7910f963a849de499f20a/testFamily: creating writer at 1732469919481Flushing 79a8c8dba8d7910f963a849de499f20a/testFamily: appending metadata at 1732469919495 (+14 ms)Flushing 79a8c8dba8d7910f963a849de499f20a/testFamily: closing flushed file at 1732469919495Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c22476e: reopening flushed file at 1732469919917 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 79a8c8dba8d7910f963a849de499f20a in 448ms, sequenceid=8, compaction requested=false at 1732469919928 (+11 ms)Writing region close event to WAL at 1732469919929 (+1 ms)Running coprocessor post-close hooks at 1732469919934 (+5 ms)Closed at 1732469919934 2024-11-24T17:38:39,934 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732469875272.79a8c8dba8d7910f963a849de499f20a. 2024-11-24T17:38:39,934 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5cb83b0a7cbc6abd6f9b140957151cf9, disabling compactions & flushes 2024-11-24T17:38:39,934 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:38:39,934 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:38:39,934 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. after waiting 0 ms 2024-11-24T17:38:39,934 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:38:39,934 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5cb83b0a7cbc6abd6f9b140957151cf9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:39,949 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/.tmp/testFamily/b26731ad37914e7ca761ec90f9adfaba is 61, key is row-0/testFamily:testQualifier/1732469866124/Put/seqid=0 2024-11-24T17:38:39,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741863_1039 (size=5049) 2024-11-24T17:38:40,118 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:40,319 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 5cb83b0a7cbc6abd6f9b140957151cf9, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:40,354 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/.tmp/testFamily/b26731ad37914e7ca761ec90f9adfaba 2024-11-24T17:38:40,364 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/.tmp/testFamily/b26731ad37914e7ca761ec90f9adfaba as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/testFamily/b26731ad37914e7ca761ec90f9adfaba 2024-11-24T17:38:40,373 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/testFamily/b26731ad37914e7ca761ec90f9adfaba, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:40,374 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5cb83b0a7cbc6abd6f9b140957151cf9 in 440ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:40,378 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnNextAsync/5cb83b0a7cbc6abd6f9b140957151cf9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:40,379 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:38:40,379 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5cb83b0a7cbc6abd6f9b140957151cf9: Waiting for close lock at 1732469919934Running coprocessor pre-close hooks at 1732469919934Disabling compacts and flushes for region at 1732469919934Disabling writes for close at 1732469919934Obtaining lock to block concurrent updates at 1732469919934Preparing flush snapshotting stores in 5cb83b0a7cbc6abd6f9b140957151cf9 at 1732469919934Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469919935 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. at 1732469919935Flushing 5cb83b0a7cbc6abd6f9b140957151cf9/testFamily: creating writer at 1732469919935Flushing 5cb83b0a7cbc6abd6f9b140957151cf9/testFamily: appending metadata at 1732469919949 (+14 ms)Flushing 5cb83b0a7cbc6abd6f9b140957151cf9/testFamily: closing flushed file at 1732469919949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56a186a5: reopening flushed file at 1732469920363 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5cb83b0a7cbc6abd6f9b140957151cf9 in 440ms, sequenceid=8, compaction requested=false at 1732469920374 (+11 ms)Writing region close event to WAL at 1732469920374Running coprocessor post-close hooks at 1732469920379 (+5 ms)Closed at 1732469920379 2024-11-24T17:38:40,379 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732469863065.5cb83b0a7cbc6abd6f9b140957151cf9. 2024-11-24T17:38:40,379 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ad9c9ae020b5952e8b085cac9777211, disabling compactions & flushes 2024-11-24T17:38:40,379 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:38:40,380 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:38:40,380 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. after waiting 0 ms 2024-11-24T17:38:40,380 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:38:40,380 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4ad9c9ae020b5952e8b085cac9777211 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:40,395 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/.tmp/testFamily/e20e0c8aa1274d14bc4625cc298dd5c8 is 61, key is row-0/testFamily:testQualifier/1732469875212/Put/seqid=0 2024-11-24T17:38:40,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741864_1040 (size=5049) 2024-11-24T17:38:40,519 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:40,720 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 4ad9c9ae020b5952e8b085cac9777211, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:40,802 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/.tmp/testFamily/e20e0c8aa1274d14bc4625cc298dd5c8 2024-11-24T17:38:40,817 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/.tmp/testFamily/e20e0c8aa1274d14bc4625cc298dd5c8 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/testFamily/e20e0c8aa1274d14bc4625cc298dd5c8 2024-11-24T17:38:40,825 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/testFamily/e20e0c8aa1274d14bc4625cc298dd5c8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:40,826 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4ad9c9ae020b5952e8b085cac9777211 in 446ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:40,831 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testRetryOutOfOrderScannerNextException/4ad9c9ae020b5952e8b085cac9777211/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:40,831 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:38:40,831 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ad9c9ae020b5952e8b085cac9777211: Waiting for close lock at 1732469920379Running coprocessor pre-close hooks at 1732469920379Disabling compacts and flushes for region at 1732469920379Disabling writes for close at 1732469920380 (+1 ms)Obtaining lock to block concurrent updates at 1732469920380Preparing flush snapshotting stores in 4ad9c9ae020b5952e8b085cac9777211 at 1732469920380Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469920380Flushing stores of testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. at 1732469920381 (+1 ms)Flushing 4ad9c9ae020b5952e8b085cac9777211/testFamily: creating writer at 1732469920381Flushing 4ad9c9ae020b5952e8b085cac9777211/testFamily: appending metadata at 1732469920395 (+14 ms)Flushing 4ad9c9ae020b5952e8b085cac9777211/testFamily: closing flushed file at 1732469920395Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a6c0d86: reopening flushed file at 1732469920816 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4ad9c9ae020b5952e8b085cac9777211 in 446ms, sequenceid=8, compaction requested=false at 1732469920826 (+10 ms)Writing region close event to WAL at 1732469920827 (+1 ms)Running coprocessor post-close hooks at 1732469920831 (+4 ms)Closed at 1732469920831 2024-11-24T17:38:40,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732469872165.4ad9c9ae020b5952e8b085cac9777211. 2024-11-24T17:38:40,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9838185cebea3a0872d5c8915ed6110b, disabling compactions & flushes 2024-11-24T17:38:40,832 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:40,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:40,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. after waiting 0 ms 2024-11-24T17:38:40,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:40,832 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9838185cebea3a0872d5c8915ed6110b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:40,847 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/.tmp/testFamily/3fa3ca76ed1f48938464b64a38882c50 is 61, key is row-0/testFamily:testQualifier/1732469884428/Put/seqid=0 2024-11-24T17:38:40,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741865_1041 (size=5049) 2024-11-24T17:38:40,920 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-24T17:38:40,920 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1325): Online Regions={9838185cebea3a0872d5c8915ed6110b=testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b., 844f29b4217aafa54fc11bf6d2a46339=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., 2598aa5bbf9a1a0e256d89173660521c=testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.} 2024-11-24T17:38:40,920 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:41,121 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 844f29b4217aafa54fc11bf6d2a46339, 9838185cebea3a0872d5c8915ed6110b 2024-11-24T17:38:41,254 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/.tmp/testFamily/3fa3ca76ed1f48938464b64a38882c50 2024-11-24T17:38:41,269 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/.tmp/testFamily/3fa3ca76ed1f48938464b64a38882c50 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/testFamily/3fa3ca76ed1f48938464b64a38882c50 2024-11-24T17:38:41,277 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/testFamily/3fa3ca76ed1f48938464b64a38882c50, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:41,278 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9838185cebea3a0872d5c8915ed6110b in 446ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:41,283 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNextAsync/9838185cebea3a0872d5c8915ed6110b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:41,283 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:41,283 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9838185cebea3a0872d5c8915ed6110b: Waiting for close lock at 1732469920832Running coprocessor pre-close hooks at 1732469920832Disabling compacts and flushes for region at 1732469920832Disabling writes for close at 1732469920832Obtaining lock to block concurrent updates at 1732469920832Preparing flush snapshotting stores in 9838185cebea3a0872d5c8915ed6110b at 1732469920832Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469920832Flushing stores of testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. at 1732469920833 (+1 ms)Flushing 9838185cebea3a0872d5c8915ed6110b/testFamily: creating writer at 1732469920833Flushing 9838185cebea3a0872d5c8915ed6110b/testFamily: appending metadata at 1732469920846 (+13 ms)Flushing 9838185cebea3a0872d5c8915ed6110b/testFamily: closing flushed file at 1732469920846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f9ea2e: reopening flushed file at 1732469921267 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9838185cebea3a0872d5c8915ed6110b in 446ms, sequenceid=8, compaction requested=false at 1732469921278 (+11 ms)Writing region close event to WAL at 1732469921279 (+1 ms)Running coprocessor post-close hooks at 1732469921283 (+4 ms)Closed at 1732469921283 2024-11-24T17:38:41,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732469881379.9838185cebea3a0872d5c8915ed6110b. 2024-11-24T17:38:41,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 844f29b4217aafa54fc11bf6d2a46339, disabling compactions & flushes 2024-11-24T17:38:41,284 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:41,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:41,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. after waiting 0 ms 2024-11-24T17:38:41,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:41,284 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 844f29b4217aafa54fc11bf6d2a46339 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:41,301 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/.tmp/testFamily/1a75952aee7240678157236d4ba48e42 is 61, key is row-0/testFamily:testQualifier/1732469896604/Put/seqid=0 2024-11-24T17:38:41,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741866_1042 (size=5049) 2024-11-24T17:38:41,321 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:41,522 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:41,708 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/.tmp/testFamily/1a75952aee7240678157236d4ba48e42 2024-11-24T17:38:41,722 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c, 844f29b4217aafa54fc11bf6d2a46339 2024-11-24T17:38:41,723 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/.tmp/testFamily/1a75952aee7240678157236d4ba48e42 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/testFamily/1a75952aee7240678157236d4ba48e42 2024-11-24T17:38:41,730 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/testFamily/1a75952aee7240678157236d4ba48e42, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:41,731 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 844f29b4217aafa54fc11bf6d2a46339 in 447ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:41,737 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/844f29b4217aafa54fc11bf6d2a46339/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:41,738 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:41,738 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 844f29b4217aafa54fc11bf6d2a46339: Waiting for close lock at 1732469921284Running coprocessor pre-close hooks at 1732469921284Disabling compacts and flushes for region at 1732469921284Disabling writes for close at 1732469921284Obtaining lock to block concurrent updates at 1732469921284Preparing flush snapshotting stores in 844f29b4217aafa54fc11bf6d2a46339 at 1732469921284Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469921284Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. at 1732469921285 (+1 ms)Flushing 844f29b4217aafa54fc11bf6d2a46339/testFamily: creating writer at 1732469921285Flushing 844f29b4217aafa54fc11bf6d2a46339/testFamily: appending metadata at 1732469921300 (+15 ms)Flushing 844f29b4217aafa54fc11bf6d2a46339/testFamily: closing flushed file at 1732469921300Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bddf0d4: reopening flushed file at 1732469921722 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 844f29b4217aafa54fc11bf6d2a46339 in 447ms, sequenceid=8, compaction requested=false at 1732469921731 (+9 ms)Writing region close event to WAL at 1732469921732 (+1 ms)Running coprocessor post-close hooks at 1732469921737 (+5 ms)Closed at 1732469921737 2024-11-24T17:38:41,738 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732469893565.844f29b4217aafa54fc11bf6d2a46339. 2024-11-24T17:38:41,738 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2598aa5bbf9a1a0e256d89173660521c, disabling compactions & flushes 2024-11-24T17:38:41,738 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:41,738 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:41,738 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. after waiting 0 ms 2024-11-24T17:38:41,738 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:41,738 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2598aa5bbf9a1a0e256d89173660521c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T17:38:41,754 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/.tmp/testFamily/460eef58b81e420fa9b7b9d38581d4a3 is 61, key is row-0/testFamily:testQualifier/1732469890505/Put/seqid=0 2024-11-24T17:38:41,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741867_1043 (size=5049) 2024-11-24T17:38:41,922 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T17:38:41,922 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1325): Online Regions={2598aa5bbf9a1a0e256d89173660521c=testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c.} 2024-11-24T17:38:41,923 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:42,123 DEBUG [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1351): Waiting on 2598aa5bbf9a1a0e256d89173660521c 2024-11-24T17:38:42,161 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/.tmp/testFamily/460eef58b81e420fa9b7b9d38581d4a3 2024-11-24T17:38:42,174 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/.tmp/testFamily/460eef58b81e420fa9b7b9d38581d4a3 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/testFamily/460eef58b81e420fa9b7b9d38581d4a3 2024-11-24T17:38:42,180 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/testFamily/460eef58b81e420fa9b7b9d38581d4a3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T17:38:42,181 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2598aa5bbf9a1a0e256d89173660521c in 443ms, sequenceid=8, compaction requested=false 2024-11-24T17:38:42,186 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/data/default/testNormalScanTimeoutOnNext/2598aa5bbf9a1a0e256d89173660521c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T17:38:42,187 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:42,187 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2598aa5bbf9a1a0e256d89173660521c: Waiting for close lock at 1732469921738Running coprocessor pre-close hooks at 1732469921738Disabling compacts and flushes for region at 1732469921738Disabling writes for close at 1732469921738Obtaining lock to block concurrent updates at 1732469921738Preparing flush snapshotting stores in 2598aa5bbf9a1a0e256d89173660521c at 1732469921738Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732469921738Flushing stores of testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. at 1732469921739 (+1 ms)Flushing 2598aa5bbf9a1a0e256d89173660521c/testFamily: creating writer at 1732469921739Flushing 2598aa5bbf9a1a0e256d89173660521c/testFamily: appending metadata at 1732469921754 (+15 ms)Flushing 2598aa5bbf9a1a0e256d89173660521c/testFamily: closing flushed file at 1732469921754Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70eac205: reopening flushed file at 1732469922173 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2598aa5bbf9a1a0e256d89173660521c in 443ms, sequenceid=8, compaction requested=false at 1732469922181 (+8 ms)Writing region close event to WAL at 1732469922182 (+1 ms)Running coprocessor post-close hooks at 1732469922187 (+5 ms)Closed at 1732469922187 2024-11-24T17:38:42,187 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732469887469.2598aa5bbf9a1a0e256d89173660521c. 2024-11-24T17:38:42,324 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(976): stopping server a23eb99f71f0,34897,1732469786783; all regions closed. 2024-11-24T17:38:42,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741834_1010 (size=19479) 2024-11-24T17:38:42,341 DEBUG [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/oldWALs 2024-11-24T17:38:42,341 INFO [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a23eb99f71f0%2C34897%2C1732469786783.meta:.meta(num 1732469850373) 2024-11-24T17:38:42,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741833_1009 (size=20031) 2024-11-24T17:38:42,347 DEBUG [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/oldWALs 2024-11-24T17:38:42,347 INFO [RS:0;a23eb99f71f0:34897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a23eb99f71f0%2C34897%2C1732469786783:(num 1732469849953) 2024-11-24T17:38:42,347 DEBUG [RS:0;a23eb99f71f0:34897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:38:42,347 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:38:42,348 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T17:38:42,348 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.ChoreService(370): Chore service for: regionserver/a23eb99f71f0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T17:38:42,348 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T17:38:42,348 INFO [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T17:38:42,350 INFO [RS:0;a23eb99f71f0:34897 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34897 2024-11-24T17:38:42,426 INFO [regionserver/a23eb99f71f0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:38:42,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a23eb99f71f0,34897,1732469786783 2024-11-24T17:38:42,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T17:38:42,515 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T17:38:42,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a23eb99f71f0,34897,1732469786783] 2024-11-24T17:38:42,535 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a23eb99f71f0,34897,1732469786783 already deleted, retry=false 2024-11-24T17:38:42,535 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a23eb99f71f0,34897,1732469786783 expired; onlineServers=0 2024-11-24T17:38:42,535 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a23eb99f71f0,33353,1732469785982' ***** 2024-11-24T17:38:42,535 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T17:38:42,536 INFO [M:0;a23eb99f71f0:33353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T17:38:42,536 INFO [M:0;a23eb99f71f0:33353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T17:38:42,536 DEBUG [M:0;a23eb99f71f0:33353 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T17:38:42,537 DEBUG [M:0;a23eb99f71f0:33353 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T17:38:42,537 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T17:38:42,537 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.small.0-1732469789705 {}] cleaner.HFileCleaner(306): Exit Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.small.0-1732469789705,5,FailOnTimeoutGroup] 2024-11-24T17:38:42,537 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.large.0-1732469789704 {}] cleaner.HFileCleaner(306): Exit Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.large.0-1732469789704,5,FailOnTimeoutGroup] 2024-11-24T17:38:42,537 INFO [M:0;a23eb99f71f0:33353 {}] hbase.ChoreService(370): Chore service for: master/a23eb99f71f0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T17:38:42,538 INFO [M:0;a23eb99f71f0:33353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T17:38:42,538 DEBUG [M:0;a23eb99f71f0:33353 {}] master.HMaster(1795): Stopping service threads 2024-11-24T17:38:42,538 INFO [M:0;a23eb99f71f0:33353 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T17:38:42,538 INFO [M:0;a23eb99f71f0:33353 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:38:42,539 INFO [M:0;a23eb99f71f0:33353 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:38:42,540 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T17:38:42,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T17:38:42,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:38:42,546 DEBUG [M:0;a23eb99f71f0:33353 {}] zookeeper.ZKUtil(347): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T17:38:42,546 WARN [M:0;a23eb99f71f0:33353 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T17:38:42,547 INFO [M:0;a23eb99f71f0:33353 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/.lastflushedseqids 2024-11-24T17:38:42,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741868_1044 (size=649) 2024-11-24T17:38:42,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:38:42,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x1016deafdc30001, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:38:42,627 INFO [RS:0;a23eb99f71f0:34897 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T17:38:42,627 INFO [RS:0;a23eb99f71f0:34897 {}] regionserver.HRegionServer(1031): Exiting; stopping=a23eb99f71f0,34897,1732469786783; zookeeper connection closed. 2024-11-24T17:38:42,628 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1988bfd7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1988bfd7 2024-11-24T17:38:42,630 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T17:38:42,963 INFO [M:0;a23eb99f71f0:33353 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:38:42,964 INFO [M:0;a23eb99f71f0:33353 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T17:38:42,965 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T17:38:42,965 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:38:42,965 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:38:42,965 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T17:38:42,965 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:38:42,966 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.13 KB heapSize=191.01 KB 2024-11-24T17:38:42,991 DEBUG [M:0;a23eb99f71f0:33353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb56e256343342989c9f3db7c39922bf is 82, key is hbase:meta,,1/info:regioninfo/1732469850456/Put/seqid=0 2024-11-24T17:38:42,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741869_1045 (size=5672) 2024-11-24T17:38:43,399 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb56e256343342989c9f3db7c39922bf 2024-11-24T17:38:43,433 DEBUG [M:0;a23eb99f71f0:33353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51b73a4d63534af6aa61b1ace99f6596 is 813, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732469852062/Put/seqid=0 2024-11-24T17:38:43,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741870_1046 (size=14082) 2024-11-24T17:38:43,553 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:38:43,841 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.58 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51b73a4d63534af6aa61b1ace99f6596 2024-11-24T17:38:43,855 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 51b73a4d63534af6aa61b1ace99f6596 2024-11-24T17:38:43,873 DEBUG [M:0;a23eb99f71f0:33353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/265a605c8b824dc3a9ea0c23d5223437 is 69, key is a23eb99f71f0,34897,1732469786783/rs:state/1732469819726/Put/seqid=0 2024-11-24T17:38:43,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741871_1047 (size=5156) 2024-11-24T17:38:44,279 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/265a605c8b824dc3a9ea0c23d5223437 2024-11-24T17:38:44,295 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb56e256343342989c9f3db7c39922bf as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb56e256343342989c9f3db7c39922bf 2024-11-24T17:38:44,301 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb56e256343342989c9f3db7c39922bf, entries=8, sequenceid=317, filesize=5.5 K 2024-11-24T17:38:44,302 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51b73a4d63534af6aa61b1ace99f6596 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/51b73a4d63534af6aa61b1ace99f6596 2024-11-24T17:38:44,308 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 51b73a4d63534af6aa61b1ace99f6596 2024-11-24T17:38:44,309 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/51b73a4d63534af6aa61b1ace99f6596, entries=33, sequenceid=317, filesize=13.8 K 2024-11-24T17:38:44,310 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/265a605c8b824dc3a9ea0c23d5223437 as hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/265a605c8b824dc3a9ea0c23d5223437 2024-11-24T17:38:44,316 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38651/user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/265a605c8b824dc3a9ea0c23d5223437, entries=1, sequenceid=317, filesize=5.0 K 2024-11-24T17:38:44,317 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.13 KB/167047, heapSize ~190.71 KB/195288, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1352ms, sequenceid=317, compaction requested=false 2024-11-24T17:38:44,320 INFO [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:38:44,320 DEBUG [M:0;a23eb99f71f0:33353 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732469922965Disabling compacts and flushes for region at 1732469922965Disabling writes for close at 1732469922965Obtaining lock to block concurrent updates at 1732469922966 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732469922966Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167047, getHeapSize=195528, getOffHeapSize=0, getCellsCount=364 at 1732469922967 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732469922969 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732469922969Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732469922991 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732469922991Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732469923415 (+424 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732469923433 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732469923433Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732469923855 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732469923872 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732469923872Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@167642e0: reopening flushed file at 1732469924293 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e2ca511: reopening flushed file at 1732469924301 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b953eba: reopening flushed file at 1732469924309 (+8 ms)Finished flush of dataSize ~163.13 KB/167047, heapSize ~190.71 KB/195288, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1352ms, sequenceid=317, compaction requested=false at 1732469924318 (+9 ms)Writing region close event to WAL at 1732469924320 (+2 ms)Closed at 1732469924320 2024-11-24T17:38:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36247 is added to blk_1073741830_1006 (size=188259) 2024-11-24T17:38:44,323 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/02a881ac-1d51-c61c-28ac-ab545cfe6e4e/MasterData/WALs/a23eb99f71f0,33353,1732469785982/a23eb99f71f0%2C33353%2C1732469785982.1732469788950 not finished, retry = 0 2024-11-24T17:38:44,425 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T17:38:44,425 INFO [M:0;a23eb99f71f0:33353 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T17:38:44,425 INFO [M:0;a23eb99f71f0:33353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33353 2024-11-24T17:38:44,425 INFO [M:0;a23eb99f71f0:33353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T17:38:44,639 INFO [M:0;a23eb99f71f0:33353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T17:38:44,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:38:44,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33353-0x1016deafdc30000, quorum=127.0.0.1:59932, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:38:44,644 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bc105ee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T17:38:44,646 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@360caa46{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T17:38:44,646 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T17:38:44,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30fd9473{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T17:38:44,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f7444f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/hadoop.log.dir/,STOPPED} 2024-11-24T17:38:44,650 WARN [BP-500573805-172.17.0.3-1732469782164 heartbeating to localhost/127.0.0.1:38651 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T17:38:44,651 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T17:38:44,651 WARN [BP-500573805-172.17.0.3-1732469782164 heartbeating to localhost/127.0.0.1:38651 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-500573805-172.17.0.3-1732469782164 (Datanode Uuid df4dbb39-e07a-4574-bd92-bc76e4ec5c2e) service to localhost/127.0.0.1:38651 2024-11-24T17:38:44,651 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T17:38:44,652 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/data/data1/current/BP-500573805-172.17.0.3-1732469782164 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T17:38:44,652 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/cluster_2509d372-606e-8ff3-d959-5e2657a6dcc3/data/data2/current/BP-500573805-172.17.0.3-1732469782164 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T17:38:44,653 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T17:38:44,659 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ce6ffde{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T17:38:44,660 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5c90255f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T17:38:44,660 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T17:38:44,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3959e81b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T17:38:44,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3bcd68eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5431730c-6f23-256a-5494-38291b6afeb1/hadoop.log.dir/,STOPPED} 2024-11-24T17:38:44,666 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T17:38:44,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down