2024-12-05 09:16:41,355 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-05 09:16:41,373 main DEBUG Took 0.016160 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 09:16:41,374 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 09:16:41,374 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 09:16:41,375 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 09:16:41,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,384 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 09:16:41,396 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,399 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,399 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,400 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,401 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,402 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,402 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,403 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,404 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,405 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,406 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,407 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,408 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,409 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,410 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 09:16:41,411 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,411 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 09:16:41,413 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 09:16:41,415 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 09:16:41,418 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 09:16:41,419 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 09:16:41,420 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 09:16:41,421 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 09:16:41,429 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 09:16:41,431 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 09:16:41,433 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 09:16:41,434 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 09:16:41,434 main DEBUG createAppenders(={Console}) 2024-12-05 09:16:41,435 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-05 09:16:41,435 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-05 09:16:41,435 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-05 09:16:41,436 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 09:16:41,436 main DEBUG OutputStream closed 2024-12-05 09:16:41,436 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 09:16:41,437 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 09:16:41,437 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-05 09:16:41,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 09:16:41,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 09:16:41,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 09:16:41,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 09:16:41,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 09:16:41,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 09:16:41,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 09:16:41,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 09:16:41,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 09:16:41,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 09:16:41,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 09:16:41,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 09:16:41,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 09:16:41,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 09:16:41,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 09:16:41,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 09:16:41,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 09:16:41,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 09:16:41,511 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 09:16:41,512 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-05 09:16:41,512 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 09:16:41,513 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-05T09:16:41,525 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-05 09:16:41,527 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 09:16:41,528 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T09:16:41,744 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034 2024-12-05T09:16:41,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T09:16:41,771 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972, deleteOnExit=true 2024-12-05T09:16:41,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T09:16:41,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/test.cache.data in system properties and HBase conf 2024-12-05T09:16:41,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T09:16:41,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/hadoop.log.dir in system properties and HBase conf 2024-12-05T09:16:41,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T09:16:41,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T09:16:41,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T09:16:41,854 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T09:16:41,940 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T09:16:41,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T09:16:41,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T09:16:41,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T09:16:41,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T09:16:41,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T09:16:41,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T09:16:41,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T09:16:41,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T09:16:41,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T09:16:41,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/nfs.dump.dir in system properties and HBase conf 2024-12-05T09:16:41,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/java.io.tmpdir in system properties and HBase conf 2024-12-05T09:16:41,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T09:16:41,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T09:16:41,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T09:16:42,887 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T09:16:42,955 INFO [Time-limited test {}] log.Log(170): Logging initialized @2193ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T09:16:43,018 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T09:16:43,081 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T09:16:43,102 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T09:16:43,102 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T09:16:43,104 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T09:16:43,115 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T09:16:43,117 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/hadoop.log.dir/,AVAILABLE} 2024-12-05T09:16:43,118 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T09:16:43,289 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/java.io.tmpdir/jetty-localhost-46433-hadoop-hdfs-3_4_1-tests_jar-_-any-2410018779569789832/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T09:16:43,299 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:46433} 2024-12-05T09:16:43,299 INFO [Time-limited test {}] server.Server(415): Started @2538ms 2024-12-05T09:16:43,784 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T09:16:43,790 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T09:16:43,791 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T09:16:43,791 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T09:16:43,791 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T09:16:43,792 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/hadoop.log.dir/,AVAILABLE} 2024-12-05T09:16:43,792 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T09:16:43,885 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/java.io.tmpdir/jetty-localhost-41227-hadoop-hdfs-3_4_1-tests_jar-_-any-10562339629008436490/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T09:16:43,886 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:41227} 2024-12-05T09:16:43,886 INFO [Time-limited test {}] server.Server(415): Started @3125ms 2024-12-05T09:16:43,932 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T09:16:44,691 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/data/data2/current/BP-164610373-172.17.0.3-1733390202473/current, will proceed with Du for space computation calculation, 2024-12-05T09:16:44,691 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/data/data1/current/BP-164610373-172.17.0.3-1733390202473/current, will proceed with Du for space computation calculation, 2024-12-05T09:16:44,717 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T09:16:44,761 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd26025d8a937725b with lease ID 0x4f01199afb1fb3ac: Processing first storage report for DS-e408501b-3643-4c05-be0d-d1071f4de7b7 from datanode DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b19e8739-c778-4b27-b5c8-25196c8e86b6, infoPort=38905, infoSecurePort=0, ipcPort=34977, storageInfo=lv=-57;cid=testClusterID;nsid=1910553971;c=1733390202473) 2024-12-05T09:16:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd26025d8a937725b with lease ID 0x4f01199afb1fb3ac: from storage DS-e408501b-3643-4c05-be0d-d1071f4de7b7 node DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b19e8739-c778-4b27-b5c8-25196c8e86b6, infoPort=38905, infoSecurePort=0, ipcPort=34977, storageInfo=lv=-57;cid=testClusterID;nsid=1910553971;c=1733390202473), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T09:16:44,763 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd26025d8a937725b with lease ID 0x4f01199afb1fb3ac: Processing first storage report for DS-4fb25d34-904c-4e4d-ac81-c96fefd7adcd from datanode DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b19e8739-c778-4b27-b5c8-25196c8e86b6, infoPort=38905, infoSecurePort=0, ipcPort=34977, storageInfo=lv=-57;cid=testClusterID;nsid=1910553971;c=1733390202473) 2024-12-05T09:16:44,763 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd26025d8a937725b with lease ID 0x4f01199afb1fb3ac: from storage DS-4fb25d34-904c-4e4d-ac81-c96fefd7adcd node DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b19e8739-c778-4b27-b5c8-25196c8e86b6, infoPort=38905, infoSecurePort=0, ipcPort=34977, storageInfo=lv=-57;cid=testClusterID;nsid=1910553971;c=1733390202473), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T09:16:44,806 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034 2024-12-05T09:16:44,871 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/zookeeper_0, clientPort=64517, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T09:16:44,880 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64517 2024-12-05T09:16:44,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:44,892 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:45,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741825_1001 (size=7) 2024-12-05T09:16:45,495 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c with version=8 2024-12-05T09:16:45,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/hbase-staging 2024-12-05T09:16:45,573 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T09:16:45,783 INFO [Time-limited test {}] client.ConnectionUtils(128): master/33186fd147fa:0 server-side Connection retries=9 2024-12-05T09:16:45,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T09:16:45,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T09:16:45,796 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T09:16:45,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T09:16:45,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T09:16:45,910 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T09:16:45,962 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T09:16:45,971 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T09:16:45,974 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T09:16:45,997 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 10993 (auto-detected) 2024-12-05T09:16:45,998 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-05T09:16:46,017 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36919 2024-12-05T09:16:46,036 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36919 connecting to ZooKeeper ensemble=127.0.0.1:64517 2024-12-05T09:16:46,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369190x0, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T09:16:46,155 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36919-0x101a5f545600000 connected 2024-12-05T09:16:46,235 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:46,238 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:46,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T09:16:46,253 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c, hbase.cluster.distributed=false 2024-12-05T09:16:46,273 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T09:16:46,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36919 2024-12-05T09:16:46,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36919 2024-12-05T09:16:46,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36919 2024-12-05T09:16:46,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36919 2024-12-05T09:16:46,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36919 2024-12-05T09:16:46,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/33186fd147fa:0 server-side Connection retries=9 2024-12-05T09:16:46,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T09:16:46,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T09:16:46,366 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T09:16:46,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T09:16:46,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T09:16:46,368 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T09:16:46,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T09:16:46,371 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43109 2024-12-05T09:16:46,373 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43109 connecting to ZooKeeper ensemble=127.0.0.1:64517 2024-12-05T09:16:46,374 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:46,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:46,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431090x0, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T09:16:46,390 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:431090x0, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T09:16:46,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43109-0x101a5f545600001 connected 2024-12-05T09:16:46,393 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T09:16:46,401 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T09:16:46,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T09:16:46,410 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T09:16:46,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43109 2024-12-05T09:16:46,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43109 2024-12-05T09:16:46,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43109 2024-12-05T09:16:46,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43109 2024-12-05T09:16:46,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43109 2024-12-05T09:16:46,428 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;33186fd147fa:36919 2024-12-05T09:16:46,429 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/33186fd147fa,36919,1733390205638 2024-12-05T09:16:46,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T09:16:46,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T09:16:46,444 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/33186fd147fa,36919,1733390205638 2024-12-05T09:16:46,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:46,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T09:16:46,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:46,467 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T09:16:46,468 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/33186fd147fa,36919,1733390205638 from backup master directory 2024-12-05T09:16:46,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T09:16:46,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/33186fd147fa,36919,1733390205638 2024-12-05T09:16:46,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T09:16:46,476 WARN [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T09:16:46,476 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=33186fd147fa,36919,1733390205638 2024-12-05T09:16:46,478 INFO [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T09:16:46,479 INFO [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T09:16:46,533 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/hbase.id] with ID: 892379cf-bd83-4299-aa13-e24a321b97ce 2024-12-05T09:16:46,533 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/.tmp/hbase.id 2024-12-05T09:16:46,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741826_1002 (size=42) 2024-12-05T09:16:46,948 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/.tmp/hbase.id]:[hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/hbase.id] 2024-12-05T09:16:47,003 INFO [master/33186fd147fa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T09:16:47,008 INFO [master/33186fd147fa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T09:16:47,025 INFO [master/33186fd147fa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-05T09:16:47,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:47,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:47,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741827_1003 (size=196) 2024-12-05T09:16:47,500 INFO [master/33186fd147fa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:16:47,504 INFO [master/33186fd147fa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T09:16:47,522 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T09:16:47,527 INFO [master/33186fd147fa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T09:16:47,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741828_1004 (size=1189) 2024-12-05T09:16:47,978 INFO [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store 2024-12-05T09:16:47,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741829_1005 (size=34) 2024-12-05T09:16:48,406 INFO [master/33186fd147fa:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T09:16:48,411 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:16:48,413 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T09:16:48,413 INFO [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:16:48,413 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:16:48,415 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T09:16:48,415 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:16:48,416 INFO [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:16:48,417 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733390208412Disabling compacts and flushes for region at 1733390208412Disabling writes for close at 1733390208415 (+3 ms)Writing region close event to WAL at 1733390208415Closed at 1733390208415 2024-12-05T09:16:48,419 WARN [master/33186fd147fa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/.initializing 2024-12-05T09:16:48,419 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/WALs/33186fd147fa,36919,1733390205638 2024-12-05T09:16:48,426 INFO [master/33186fd147fa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T09:16:48,439 INFO [master/33186fd147fa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=33186fd147fa%2C36919%2C1733390205638, suffix=, logDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/WALs/33186fd147fa,36919,1733390205638, archiveDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/oldWALs, maxLogs=10 2024-12-05T09:16:48,459 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/WALs/33186fd147fa,36919,1733390205638/33186fd147fa%2C36919%2C1733390205638.1733390208443, exclude list is [], retry=0 2024-12-05T09:16:48,473 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40207,DS-e408501b-3643-4c05-be0d-d1071f4de7b7,DISK] 2024-12-05T09:16:48,475 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T09:16:48,511 INFO [master/33186fd147fa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/WALs/33186fd147fa,36919,1733390205638/33186fd147fa%2C36919%2C1733390205638.1733390208443 2024-12-05T09:16:48,512 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38905:38905)] 2024-12-05T09:16:48,512 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:16:48,513 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:16:48,517 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,519 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T09:16:48,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:48,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:16:48,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T09:16:48,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:48,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:16:48,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T09:16:48,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:48,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:16:48,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T09:16:48,589 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:48,590 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:16:48,590 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,594 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,595 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,600 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,601 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,605 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T09:16:48,610 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T09:16:48,614 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:16:48,616 INFO [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72567800, jitterRate=0.08134448528289795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T09:16:48,621 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733390208529Initializing all the Stores at 1733390208531 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390208532 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390208532Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390208532Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390208532Cleaning up temporary data from old regions at 1733390208601 (+69 ms)Region opened successfully at 1733390208621 (+20 ms) 2024-12-05T09:16:48,622 INFO [master/33186fd147fa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T09:16:48,652 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c57f1e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=33186fd147fa/172.17.0.3:0 2024-12-05T09:16:48,680 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T09:16:48,692 INFO [master/33186fd147fa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T09:16:48,692 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T09:16:48,695 INFO [master/33186fd147fa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T09:16:48,696 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T09:16:48,703 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T09:16:48,703 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T09:16:48,728 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T09:16:48,739 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T09:16:48,807 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T09:16:48,810 INFO [master/33186fd147fa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T09:16:48,811 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T09:16:48,824 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T09:16:48,827 INFO [master/33186fd147fa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T09:16:48,832 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T09:16:48,841 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T09:16:48,843 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T09:16:48,849 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T09:16:48,871 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T09:16:48,880 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T09:16:48,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T09:16:48,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T09:16:48,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:48,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:48,896 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=33186fd147fa,36919,1733390205638, sessionid=0x101a5f545600000, setting cluster-up flag (Was=false) 2024-12-05T09:16:48,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:48,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:48,950 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T09:16:48,953 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=33186fd147fa,36919,1733390205638 2024-12-05T09:16:48,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:48,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:16:49,000 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T09:16:49,004 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=33186fd147fa,36919,1733390205638 2024-12-05T09:16:49,014 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T09:16:49,019 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(746): ClusterId : 892379cf-bd83-4299-aa13-e24a321b97ce 2024-12-05T09:16:49,022 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T09:16:49,035 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T09:16:49,035 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T09:16:49,042 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T09:16:49,043 DEBUG [RS:0;33186fd147fa:43109 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53a0b6aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=33186fd147fa/172.17.0.3:0 2024-12-05T09:16:49,059 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;33186fd147fa:43109 2024-12-05T09:16:49,062 INFO [RS:0;33186fd147fa:43109 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T09:16:49,063 INFO [RS:0;33186fd147fa:43109 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T09:16:49,063 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T09:16:49,066 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(2659): reportForDuty to master=33186fd147fa,36919,1733390205638 with port=43109, startcode=1733390206335 2024-12-05T09:16:49,077 DEBUG [RS:0;33186fd147fa:43109 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T09:16:49,083 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T09:16:49,095 INFO [master/33186fd147fa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T09:16:49,104 INFO [master/33186fd147fa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T09:16:49,111 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35001, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T09:16:49,112 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 33186fd147fa,36919,1733390205638 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T09:16:49,119 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36919 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T09:16:49,122 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/33186fd147fa:0, corePoolSize=5, maxPoolSize=5 2024-12-05T09:16:49,122 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/33186fd147fa:0, corePoolSize=5, maxPoolSize=5 2024-12-05T09:16:49,122 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/33186fd147fa:0, corePoolSize=5, maxPoolSize=5 2024-12-05T09:16:49,122 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/33186fd147fa:0, corePoolSize=5, maxPoolSize=5 2024-12-05T09:16:49,123 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/33186fd147fa:0, corePoolSize=10, maxPoolSize=10 2024-12-05T09:16:49,123 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:16:49,123 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/33186fd147fa:0, corePoolSize=2, maxPoolSize=2 2024-12-05T09:16:49,123 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:16:49,130 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T09:16:49,130 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733390239130 2024-12-05T09:16:49,131 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T09:16:49,132 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T09:16:49,134 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T09:16:49,136 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:49,137 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T09:16:49,138 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T09:16:49,139 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T09:16:49,139 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T09:16:49,140 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T09:16:49,143 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T09:16:49,147 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T09:16:49,147 WARN [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-12-05T09:16:49,147 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T09:16:49,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741831_1007 (size=1321) 2024-12-05T09:16:49,149 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T09:16:49,150 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T09:16:49,152 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T09:16:49,152 INFO [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T09:16:49,154 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/33186fd147fa:0:becomeActiveMaster-HFileCleaner.large.0-1733390209154,5,FailOnTimeoutGroup] 2024-12-05T09:16:49,155 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/33186fd147fa:0:becomeActiveMaster-HFileCleaner.small.0-1733390209155,5,FailOnTimeoutGroup] 2024-12-05T09:16:49,155 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T09:16:49,155 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T09:16:49,156 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T09:16:49,156 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T09:16:49,552 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T09:16:49,553 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:16:49,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741832_1008 (size=32) 2024-12-05T09:16:49,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:16:49,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T09:16:49,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T09:16:49,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:49,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:16:49,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T09:16:49,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T09:16:49,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:49,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:16:49,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T09:16:49,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T09:16:49,986 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:49,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:16:49,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T09:16:49,990 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T09:16:49,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:16:49,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:16:49,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T09:16:49,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740 2024-12-05T09:16:49,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740 2024-12-05T09:16:49,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T09:16:49,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T09:16:49,997 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T09:16:49,999 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T09:16:50,003 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:16:50,004 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71389080, jitterRate=0.06378018856048584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T09:16:50,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733390209968Initializing all the Stores at 1733390209971 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390209971Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390209971Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390209971Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390209971Cleaning up temporary data from old regions at 1733390209996 (+25 ms)Region opened successfully at 1733390210006 (+10 ms) 2024-12-05T09:16:50,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T09:16:50,007 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T09:16:50,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T09:16:50,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T09:16:50,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T09:16:50,009 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T09:16:50,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733390210007Disabling compacts and flushes for region at 1733390210007Disabling writes for close at 1733390210007Writing region close event to WAL at 1733390210008 (+1 ms)Closed at 1733390210009 (+1 ms) 2024-12-05T09:16:50,012 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T09:16:50,012 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T09:16:50,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T09:16:50,025 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T09:16:50,028 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T09:16:50,180 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:51,182 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:52,183 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:53,184 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:54,185 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:55,186 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:55,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T09:16:55,962 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T09:16:56,188 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:57,189 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:58,190 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:16:59,191 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:00,193 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:01,194 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:02,195 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:03,197 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:04,198 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:05,199 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:06,200 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:07,201 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:08,203 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:09,204 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:10,205 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:11,205 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:12,206 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:13,207 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:14,208 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:14,803 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T09:17:15,209 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:16,210 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:17,211 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:18,212 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:19,148 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(2659): reportForDuty to master=33186fd147fa,36919,1733390205638 with port=43109, startcode=1733390206335 2024-12-05T09:17:19,150 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36919 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 33186fd147fa,43109,1733390206335 2024-12-05T09:17:19,153 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36919 {}] master.ServerManager(517): Registering regionserver=33186fd147fa,43109,1733390206335 2024-12-05T09:17:19,162 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:17:19,162 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35883 2024-12-05T09:17:19,162 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T09:17:19,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T09:17:19,173 DEBUG [RS:0;33186fd147fa:43109 {}] zookeeper.ZKUtil(111): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/33186fd147fa,43109,1733390206335 2024-12-05T09:17:19,173 WARN [RS:0;33186fd147fa:43109 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T09:17:19,174 INFO [RS:0;33186fd147fa:43109 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T09:17:19,174 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335 2024-12-05T09:17:19,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [33186fd147fa,43109,1733390206335] 2024-12-05T09:17:19,202 INFO [RS:0;33186fd147fa:43109 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T09:17:19,212 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:19,213 INFO [RS:0;33186fd147fa:43109 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T09:17:19,217 INFO [RS:0;33186fd147fa:43109 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T09:17:19,217 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,218 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T09:17:19,223 INFO [RS:0;33186fd147fa:43109 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T09:17:19,225 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,225 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,225 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,225 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/33186fd147fa:0, corePoolSize=2, maxPoolSize=2 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,226 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,227 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,227 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/33186fd147fa:0, corePoolSize=1, maxPoolSize=1 2024-12-05T09:17:19,227 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/33186fd147fa:0, corePoolSize=3, maxPoolSize=3 2024-12-05T09:17:19,227 DEBUG [RS:0;33186fd147fa:43109 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/33186fd147fa:0, corePoolSize=3, maxPoolSize=3 2024-12-05T09:17:19,228 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,228 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,228 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,228 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,228 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,228 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,43109,1733390206335-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T09:17:19,248 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T09:17:19,250 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,43109,1733390206335-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,251 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,251 INFO [RS:0;33186fd147fa:43109 {}] regionserver.Replication(171): 33186fd147fa,43109,1733390206335 started 2024-12-05T09:17:19,268 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:19,269 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1482): Serving as 33186fd147fa,43109,1733390206335, RpcServer on 33186fd147fa/172.17.0.3:43109, sessionid=0x101a5f545600001 2024-12-05T09:17:19,269 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T09:17:19,270 DEBUG [RS:0;33186fd147fa:43109 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 33186fd147fa,43109,1733390206335 2024-12-05T09:17:19,270 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '33186fd147fa,43109,1733390206335' 2024-12-05T09:17:19,270 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T09:17:19,271 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T09:17:19,273 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T09:17:19,273 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T09:17:19,273 DEBUG [RS:0;33186fd147fa:43109 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 33186fd147fa,43109,1733390206335 2024-12-05T09:17:19,273 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '33186fd147fa,43109,1733390206335' 2024-12-05T09:17:19,273 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T09:17:19,276 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T09:17:19,277 DEBUG [RS:0;33186fd147fa:43109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T09:17:19,277 INFO [RS:0;33186fd147fa:43109 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T09:17:19,277 INFO [RS:0;33186fd147fa:43109 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T09:17:20,213 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:21,214 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:22,215 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:23,216 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:24,217 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:25,217 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:26,218 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:27,219 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:28,220 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:29,220 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:30,221 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:31,222 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:32,222 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:33,223 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:34,224 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:35,225 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:36,226 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:37,226 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:38,227 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:39,228 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:40,229 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:41,230 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:42,230 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:43,231 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:44,232 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:44,803 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T09:17:45,233 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:46,234 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:47,234 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:48,235 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:49,236 WARN [33186fd147fa:36919 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T09:17:49,285 INFO [RS:0;33186fd147fa:43109 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T09:17:49,289 INFO [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=33186fd147fa%2C43109%2C1733390206335, suffix=, logDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335, archiveDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/oldWALs, maxLogs=32 2024-12-05T09:17:49,304 DEBUG [RS:0;33186fd147fa:43109 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335/33186fd147fa%2C43109%2C1733390206335.1733390269291, exclude list is [], retry=0 2024-12-05T09:17:49,309 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40207,DS-e408501b-3643-4c05-be0d-d1071f4de7b7,DISK] 2024-12-05T09:17:49,312 INFO [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335/33186fd147fa%2C43109%2C1733390206335.1733390269291 2024-12-05T09:17:49,313 DEBUG [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38905:38905)] 2024-12-05T09:17:49,488 DEBUG [33186fd147fa:36919 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T09:17:49,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:17:49,504 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 33186fd147fa,43109,1733390206335, state=OPENING 2024-12-05T09:17:49,563 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T09:17:49,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:17:49,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:17:49,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T09:17:49,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T09:17:49,613 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T09:17:49,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:17:49,789 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T09:17:49,793 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54595, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T09:17:49,806 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T09:17:49,807 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T09:17:49,807 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T09:17:49,810 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=33186fd147fa%2C43109%2C1733390206335.meta, suffix=.meta, logDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335, archiveDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/oldWALs, maxLogs=32 2024-12-05T09:17:49,824 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335/33186fd147fa%2C43109%2C1733390206335.meta.1733390269812.meta, exclude list is [], retry=0 2024-12-05T09:17:49,828 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40207,DS-e408501b-3643-4c05-be0d-d1071f4de7b7,DISK] 2024-12-05T09:17:49,831 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335/33186fd147fa%2C43109%2C1733390206335.meta.1733390269812.meta 2024-12-05T09:17:49,831 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38905:38905)] 2024-12-05T09:17:49,832 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:17:49,833 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T09:17:49,835 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T09:17:49,839 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T09:17:49,843 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T09:17:49,844 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:17:49,844 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T09:17:49,844 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T09:17:49,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T09:17:49,849 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T09:17:49,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:49,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:17:49,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T09:17:49,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T09:17:49,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:49,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:17:49,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T09:17:49,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T09:17:49,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:49,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:17:49,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T09:17:49,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T09:17:49,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:49,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T09:17:49,858 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T09:17:49,859 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740 2024-12-05T09:17:49,862 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740 2024-12-05T09:17:49,865 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T09:17:49,865 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T09:17:49,866 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T09:17:49,869 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T09:17:49,871 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59366788, jitterRate=-0.11536592245101929}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T09:17:49,871 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T09:17:49,873 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733390269845Writing region info on filesystem at 1733390269845Initializing all the Stores at 1733390269847 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390269847Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390269847Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390269847Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733390269847Cleaning up temporary data from old regions at 1733390269865 (+18 ms)Running coprocessor post-open hooks at 1733390269871 (+6 ms)Region opened successfully at 1733390269873 (+2 ms) 2024-12-05T09:17:49,880 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733390269780 2024-12-05T09:17:49,892 DEBUG [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T09:17:49,893 INFO [RS_OPEN_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T09:17:49,895 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:17:49,898 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 33186fd147fa,43109,1733390206335, state=OPEN 2024-12-05T09:17:49,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T09:17:49,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T09:17:49,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T09:17:49,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T09:17:49,985 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=33186fd147fa,43109,1733390206335 2024-12-05T09:17:49,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T09:17:49,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=33186fd147fa,43109,1733390206335 in 370 msec 2024-12-05T09:17:50,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T09:17:50,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 59.9780 sec 2024-12-05T09:17:50,003 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T09:17:50,003 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T09:17:50,020 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T09:17:50,021 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=33186fd147fa,43109,1733390206335, seqNum=-1] 2024-12-05T09:17:50,041 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T09:17:50,043 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T09:17:50,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 1.02 sec 2024-12-05T09:17:50,063 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733390270063, completionTime=-1 2024-12-05T09:17:50,066 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T09:17:50,066 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T09:17:50,090 INFO [master/33186fd147fa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T09:17:50,090 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733390330090 2024-12-05T09:17:50,090 INFO [master/33186fd147fa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733390390090 2024-12-05T09:17:50,090 INFO [master/33186fd147fa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-05T09:17:50,092 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,36919,1733390205638-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,092 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,36919,1733390205638-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,092 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,36919,1733390205638-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,094 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-33186fd147fa:36919, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,094 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,095 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,100 DEBUG [master/33186fd147fa:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T09:17:50,121 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.645sec 2024-12-05T09:17:50,123 INFO [master/33186fd147fa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T09:17:50,124 INFO [master/33186fd147fa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T09:17:50,125 INFO [master/33186fd147fa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T09:17:50,125 INFO [master/33186fd147fa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T09:17:50,125 INFO [master/33186fd147fa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T09:17:50,126 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,36919,1733390205638-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T09:17:50,127 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,36919,1733390205638-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T09:17:50,135 DEBUG [master/33186fd147fa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T09:17:50,136 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T09:17:50,137 INFO [master/33186fd147fa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=33186fd147fa,36919,1733390205638-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T09:17:50,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77804624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T09:17:50,204 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T09:17:50,204 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T09:17:50,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 33186fd147fa,36919,-1 for getting cluster id 2024-12-05T09:17:50,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T09:17:50,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '892379cf-bd83-4299-aa13-e24a321b97ce' 2024-12-05T09:17:50,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T09:17:50,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "892379cf-bd83-4299-aa13-e24a321b97ce" 2024-12-05T09:17:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f26a9ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T09:17:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [33186fd147fa,36919,-1] 2024-12-05T09:17:50,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T09:17:50,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:17:50,227 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T09:17:50,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T09:17:50,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T09:17:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=33186fd147fa,43109,1733390206335, seqNum=-1] 2024-12-05T09:17:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T09:17:50,240 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T09:17:50,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=33186fd147fa,36919,1733390205638 2024-12-05T09:17:50,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68214a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T09:17:50,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 33186fd147fa,36919,-1 for getting cluster id 2024-12-05T09:17:50,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T09:17:50,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '892379cf-bd83-4299-aa13-e24a321b97ce' 2024-12-05T09:17:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T09:17:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "892379cf-bd83-4299-aa13-e24a321b97ce" 2024-12-05T09:17:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5163e2ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T09:17:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [33186fd147fa,36919,-1] 2024-12-05T09:17:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T09:17:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:17:50,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T09:17:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90cdcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T09:17:50,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=243, ProcessCount=11, AvailableMemoryMB=5282 2024-12-05T09:17:50,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T09:17:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 33186fd147fa,36919,1733390205638 2024-12-05T09:17:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2547c9ac 2024-12-05T09:17:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T09:17:50,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T09:17:50,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:17:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-05T09:17:50,354 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:17:50,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-05T09:17:50,357 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:50,360 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:17:50,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T09:17:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741835_1011 (size=428) 2024-12-05T09:17:50,781 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cd4f09cb5a3393091a8bbb291e207146, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:17:50,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741836_1012 (size=79) 2024-12-05T09:17:51,196 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:17:51,196 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing cd4f09cb5a3393091a8bbb291e207146, disabling compactions & flushes 2024-12-05T09:17:51,197 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,197 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,197 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. after waiting 0 ms 2024-12-05T09:17:51,197 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,197 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,197 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cd4f09cb5a3393091a8bbb291e207146: Waiting for close lock at 1733390271196Disabling compacts and flushes for region at 1733390271196Disabling writes for close at 1733390271197 (+1 ms)Writing region close event to WAL at 1733390271197Closed at 1733390271197 2024-12-05T09:17:51,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:17:51,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733390271201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390271201"}]},"ts":"1733390271201"} 2024-12-05T09:17:51,212 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:17:51,214 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:17:51,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390271214"}]},"ts":"1733390271214"} 2024-12-05T09:17:51,223 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-05T09:17:51,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=cd4f09cb5a3393091a8bbb291e207146, ASSIGN}] 2024-12-05T09:17:51,228 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=cd4f09cb5a3393091a8bbb291e207146, ASSIGN 2024-12-05T09:17:51,230 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=cd4f09cb5a3393091a8bbb291e207146, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:17:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T09:17:51,384 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cd4f09cb5a3393091a8bbb291e207146, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:17:51,389 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=cd4f09cb5a3393091a8bbb291e207146, ASSIGN because future has completed 2024-12-05T09:17:51,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd4f09cb5a3393091a8bbb291e207146, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:17:51,557 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,557 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cd4f09cb5a3393091a8bbb291e207146, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:17:51,558 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,558 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:17:51,559 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,559 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,562 INFO [StoreOpener-cd4f09cb5a3393091a8bbb291e207146-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,565 INFO [StoreOpener-cd4f09cb5a3393091a8bbb291e207146-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd4f09cb5a3393091a8bbb291e207146 columnFamilyName testFamily 2024-12-05T09:17:51,566 DEBUG [StoreOpener-cd4f09cb5a3393091a8bbb291e207146-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:51,567 INFO [StoreOpener-cd4f09cb5a3393091a8bbb291e207146-1 {}] regionserver.HStore(327): Store=cd4f09cb5a3393091a8bbb291e207146/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:17:51,567 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,569 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,570 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,571 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,571 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,574 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,580 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:17:51,582 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cd4f09cb5a3393091a8bbb291e207146; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68343954, jitterRate=0.018404275178909302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:17:51,583 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:17:51,584 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cd4f09cb5a3393091a8bbb291e207146: Running coprocessor pre-open hook at 1733390271559Writing region info on filesystem at 1733390271559Initializing all the Stores at 1733390271561 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390271562 (+1 ms)Cleaning up temporary data from old regions at 1733390271571 (+9 ms)Running coprocessor post-open hooks at 1733390271583 (+12 ms)Region opened successfully at 1733390271584 (+1 ms) 2024-12-05T09:17:51,587 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146., pid=6, masterSystemTime=1733390271546 2024-12-05T09:17:51,592 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,592 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:17:51,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cd4f09cb5a3393091a8bbb291e207146, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:17:51,605 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd4f09cb5a3393091a8bbb291e207146, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:17:51,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T09:17:51,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cd4f09cb5a3393091a8bbb291e207146, server=33186fd147fa,43109,1733390206335 in 217 msec 2024-12-05T09:17:51,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T09:17:51,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=cd4f09cb5a3393091a8bbb291e207146, ASSIGN in 387 msec 2024-12-05T09:17:51,618 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:17:51,618 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390271618"}]},"ts":"1733390271618"} 2024-12-05T09:17:51,621 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-05T09:17:51,622 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:17:51,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2810 sec 2024-12-05T09:17:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T09:17:53,398 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-05T09:17:53,398 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-05T09:17:53,400 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:17:53,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-05T09:17:53,408 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:17:53,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-05T09:17:53,411 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T09:17:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=33186fd147fa,43109,1733390206335, seqNum=-1] 2024-12-05T09:17:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T09:17:53,417 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T09:17:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:17:53,438 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:17:53,442 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-05T09:17:53,449 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-05T09:17:53,454 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-05T09:17:53,456 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-05T09:17:53,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3212928119170138104 2024-12-05T09:17:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390276458, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:17:53,468 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-05T09:17:53,469 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-05T09:17:53,471 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:17:53,473 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:17:53,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3212928119170138102 2024-12-05T09:17:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390276473, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:17:53,479 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:17:53,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3212928119170138101 2024-12-05T09:17:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390276478, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:17:53,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:17:53,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3212928119170138100 2024-12-05T09:17:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390276483, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:17:53,488 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:17:53,498 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RS_OPEN_REGION-regionserver/33186fd147fa:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-707245712_22 at /127.0.0.1:38648 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=223 (was 243), ProcessCount=11 (was 11), AvailableMemoryMB=5233 (was 5282) 2024-12-05T09:17:53,507 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=5233 2024-12-05T09:17:53,512 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:17:53,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-05T09:17:53,516 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:17:53,516 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:53,516 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-05T09:17:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T09:17:53,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:17:53,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741837_1013 (size=414) 2024-12-05T09:17:53,934 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 75b57c100509b5527163021652761449, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:17:53,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741838_1014 (size=65) 2024-12-05T09:17:54,348 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:17:54,349 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 75b57c100509b5527163021652761449, disabling compactions & flushes 2024-12-05T09:17:54,349 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,349 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,349 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. after waiting 0 ms 2024-12-05T09:17:54,349 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,350 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,350 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 75b57c100509b5527163021652761449: Waiting for close lock at 1733390274349Disabling compacts and flushes for region at 1733390274349Disabling writes for close at 1733390274349Writing region close event to WAL at 1733390274349Closed at 1733390274349 2024-12-05T09:17:54,355 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:17:54,355 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733390274355"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390274355"}]},"ts":"1733390274355"} 2024-12-05T09:17:54,359 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:17:54,361 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:17:54,361 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390274361"}]},"ts":"1733390274361"} 2024-12-05T09:17:54,364 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-05T09:17:54,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=75b57c100509b5527163021652761449, ASSIGN}] 2024-12-05T09:17:54,366 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=75b57c100509b5527163021652761449, ASSIGN 2024-12-05T09:17:54,367 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=75b57c100509b5527163021652761449, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:17:54,519 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=75b57c100509b5527163021652761449, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:17:54,527 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=75b57c100509b5527163021652761449, ASSIGN because future has completed 2024-12-05T09:17:54,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75b57c100509b5527163021652761449, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:17:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T09:17:54,691 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 75b57c100509b5527163021652761449, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:17:54,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 75b57c100509b5527163021652761449 2024-12-05T09:17:54,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:17:54,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,694 INFO [StoreOpener-75b57c100509b5527163021652761449-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 75b57c100509b5527163021652761449 2024-12-05T09:17:54,696 INFO [StoreOpener-75b57c100509b5527163021652761449-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75b57c100509b5527163021652761449 columnFamilyName testFamily 2024-12-05T09:17:54,696 DEBUG [StoreOpener-75b57c100509b5527163021652761449-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:17:54,697 INFO [StoreOpener-75b57c100509b5527163021652761449-1 {}] regionserver.HStore(327): Store=75b57c100509b5527163021652761449/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:17:54,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,698 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449 2024-12-05T09:17:54,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449 2024-12-05T09:17:54,700 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,700 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,705 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:17:54,706 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 75b57c100509b5527163021652761449; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71393792, jitterRate=0.06385040283203125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:17:54,706 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75b57c100509b5527163021652761449 2024-12-05T09:17:54,707 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 75b57c100509b5527163021652761449: Running coprocessor pre-open hook at 1733390274692Writing region info on filesystem at 1733390274692Initializing all the Stores at 1733390274693 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390274693Cleaning up temporary data from old regions at 1733390274700 (+7 ms)Running coprocessor post-open hooks at 1733390274706 (+6 ms)Region opened successfully at 1733390274707 (+1 ms) 2024-12-05T09:17:54,709 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., pid=9, masterSystemTime=1733390274684 2024-12-05T09:17:54,712 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,712 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:17:54,713 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=75b57c100509b5527163021652761449, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:17:54,717 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75b57c100509b5527163021652761449, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:17:54,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-05T09:17:54,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 75b57c100509b5527163021652761449, server=33186fd147fa,43109,1733390206335 in 191 msec 2024-12-05T09:17:54,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T09:17:54,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=75b57c100509b5527163021652761449, ASSIGN in 358 msec 2024-12-05T09:17:54,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:17:54,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390274727"}]},"ts":"1733390274727"} 2024-12-05T09:17:54,731 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-05T09:17:54,732 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:17:54,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2200 sec 2024-12-05T09:17:55,388 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:17:55,414 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-05T09:17:55,416 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T09:17:55,416 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-05T09:17:55,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T09:17:55,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T09:17:55,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T09:17:55,961 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T09:17:55,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-05T09:17:55,961 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T09:17:55,962 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-05T09:17:55,962 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-05T09:17:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T09:17:56,546 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-05T09:17:56,546 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-05T09:17:56,547 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:17:56,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-05T09:17:56,552 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:17:56,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-05T09:17:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:17:56,571 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:17:56,571 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-05T09:17:56,575 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T09:17:56,577 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-05T09:18:02,586 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., hostname=33186fd147fa,43109,1733390206335, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., hostname=33186fd147fa,43109,1733390206335, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=33186fd147fa:43109 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-12-05T09:18:02,587 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., hostname=33186fd147fa,43109,1733390206335, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-12-05T09:18:02,587 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., hostname=33186fd147fa,43109,1733390206335, seqNum=2 from cache 2024-12-05T09:18:02,596 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=211 (was 223), ProcessCount=11 (was 11), AvailableMemoryMB=6294 (was 5233) - AvailableMemoryMB LEAK? - 2024-12-05T09:18:02,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=211, ProcessCount=11, AvailableMemoryMB=6293 2024-12-05T09:18:02,606 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:02,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-05T09:18:02,611 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:02,611 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:02,611 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-05T09:18:02,612 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-05T09:18:02,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741839_1015 (size=412) 2024-12-05T09:18:03,026 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49bbaa611533446894035c688795e4c6, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:03,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741840_1016 (size=63) 2024-12-05T09:18:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-05T09:18:03,442 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:03,443 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 49bbaa611533446894035c688795e4c6, disabling compactions & flushes 2024-12-05T09:18:03,443 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,443 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,443 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. after waiting 0 ms 2024-12-05T09:18:03,443 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,443 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,444 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49bbaa611533446894035c688795e4c6: Waiting for close lock at 1733390283443Disabling compacts and flushes for region at 1733390283443Disabling writes for close at 1733390283443Writing region close event to WAL at 1733390283443Closed at 1733390283443 2024-12-05T09:18:03,448 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:03,449 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733390283449"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390283449"}]},"ts":"1733390283449"} 2024-12-05T09:18:03,453 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:03,455 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:03,456 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390283455"}]},"ts":"1733390283455"} 2024-12-05T09:18:03,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-05T09:18:03,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=49bbaa611533446894035c688795e4c6, ASSIGN}] 2024-12-05T09:18:03,461 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=49bbaa611533446894035c688795e4c6, ASSIGN 2024-12-05T09:18:03,463 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=49bbaa611533446894035c688795e4c6, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:03,615 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=49bbaa611533446894035c688795e4c6, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:03,623 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=49bbaa611533446894035c688795e4c6, ASSIGN because future has completed 2024-12-05T09:18:03,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49bbaa611533446894035c688795e4c6, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-05T09:18:03,791 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 49bbaa611533446894035c688795e4c6, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:03,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:03,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,795 INFO [StoreOpener-49bbaa611533446894035c688795e4c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,797 INFO [StoreOpener-49bbaa611533446894035c688795e4c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bbaa611533446894035c688795e4c6 columnFamilyName testFamily 2024-12-05T09:18:03,798 DEBUG [StoreOpener-49bbaa611533446894035c688795e4c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:03,799 INFO [StoreOpener-49bbaa611533446894035c688795e4c6-1 {}] regionserver.HStore(327): Store=49bbaa611533446894035c688795e4c6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:03,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,800 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,809 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:03,810 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 49bbaa611533446894035c688795e4c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61853643, jitterRate=-0.07830889523029327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:03,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:03,811 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 49bbaa611533446894035c688795e4c6: Running coprocessor pre-open hook at 1733390283792Writing region info on filesystem at 1733390283792Initializing all the Stores at 1733390283794 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390283794Cleaning up temporary data from old regions at 1733390283802 (+8 ms)Running coprocessor post-open hooks at 1733390283810 (+8 ms)Region opened successfully at 1733390283811 (+1 ms) 2024-12-05T09:18:03,813 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6., pid=12, masterSystemTime=1733390283780 2024-12-05T09:18:03,817 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,817 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:03,818 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=49bbaa611533446894035c688795e4c6, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:03,821 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49bbaa611533446894035c688795e4c6, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:03,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T09:18:03,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 49bbaa611533446894035c688795e4c6, server=33186fd147fa,43109,1733390206335 in 199 msec 2024-12-05T09:18:03,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T09:18:03,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=49bbaa611533446894035c688795e4c6, ASSIGN in 368 msec 2024-12-05T09:18:03,833 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:03,833 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390283833"}]},"ts":"1733390283833"} 2024-12-05T09:18:03,837 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-05T09:18:03,839 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:03,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2320 sec 2024-12-05T09:18:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-05T09:18:05,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-05T09:18:05,648 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-05T09:18:05,649 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:05,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-05T09:18:05,659 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:05,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-05T09:18:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:05,676 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:05,677 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-05T09:18:05,680 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T09:18:05,682 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T09:18:05,684 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T09:18:05,685 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-05T09:18:05,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-05T09:18:05,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:07,418 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-05T09:18:09,326 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:18:11,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138091' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T09:18:11,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:60902 deadline: 1733390297691, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138091' does not exist 2024-12-05T09:18:11,701 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 33186fd147fa,43109,1733390206335 for closing scanner id = -3212928119170138091 for 49bbaa611533446894035c688795e4c6 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138091' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T09:18:11,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 211), ProcessCount=11 (was 11), AvailableMemoryMB=6038 (was 6293) 2024-12-05T09:18:11,723 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=6037 2024-12-05T09:18:11,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-05T09:18:11,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:11,739 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:11,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-05T09:18:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T09:18:11,742 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:11,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741841_1017 (size=423) 2024-12-05T09:18:11,807 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bf7ade1d680d2bf0f19c877d69800cad, NAME => 'testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:11,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741842_1018 (size=74) 2024-12-05T09:18:11,864 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:11,864 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing bf7ade1d680d2bf0f19c877d69800cad, disabling compactions & flushes 2024-12-05T09:18:11,865 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:11,865 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:11,865 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. after waiting 0 ms 2024-12-05T09:18:11,865 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:11,865 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:11,865 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for bf7ade1d680d2bf0f19c877d69800cad: Waiting for close lock at 1733390291864Disabling compacts and flushes for region at 1733390291864Disabling writes for close at 1733390291865 (+1 ms)Writing region close event to WAL at 1733390291865Closed at 1733390291865 2024-12-05T09:18:11,869 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:11,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733390291869"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390291869"}]},"ts":"1733390291869"} 2024-12-05T09:18:11,883 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:11,907 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:11,907 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390291907"}]},"ts":"1733390291907"} 2024-12-05T09:18:11,921 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-05T09:18:11,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bf7ade1d680d2bf0f19c877d69800cad, ASSIGN}] 2024-12-05T09:18:11,927 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bf7ade1d680d2bf0f19c877d69800cad, ASSIGN 2024-12-05T09:18:11,931 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bf7ade1d680d2bf0f19c877d69800cad, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:12,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=bf7ade1d680d2bf0f19c877d69800cad, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:12,087 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bf7ade1d680d2bf0f19c877d69800cad, ASSIGN because future has completed 2024-12-05T09:18:12,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf7ade1d680d2bf0f19c877d69800cad, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-05T09:18:12,251 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:12,251 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => bf7ade1d680d2bf0f19c877d69800cad, NAME => 'testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:12,252 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,252 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:12,252 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,252 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,255 INFO [StoreOpener-bf7ade1d680d2bf0f19c877d69800cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,261 INFO [StoreOpener-bf7ade1d680d2bf0f19c877d69800cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7ade1d680d2bf0f19c877d69800cad columnFamilyName testFamily 2024-12-05T09:18:12,262 DEBUG [StoreOpener-bf7ade1d680d2bf0f19c877d69800cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:12,264 INFO [StoreOpener-bf7ade1d680d2bf0f19c877d69800cad-1 {}] regionserver.HStore(327): Store=bf7ade1d680d2bf0f19c877d69800cad/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:12,264 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,266 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,267 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,270 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,270 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,283 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,294 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:12,296 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened bf7ade1d680d2bf0f19c877d69800cad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70859256, jitterRate=0.0558851957321167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:12,296 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:12,297 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for bf7ade1d680d2bf0f19c877d69800cad: Running coprocessor pre-open hook at 1733390292252Writing region info on filesystem at 1733390292252Initializing all the Stores at 1733390292254 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390292254Cleaning up temporary data from old regions at 1733390292270 (+16 ms)Running coprocessor post-open hooks at 1733390292296 (+26 ms)Region opened successfully at 1733390292297 (+1 ms) 2024-12-05T09:18:12,300 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad., pid=15, masterSystemTime=1733390292244 2024-12-05T09:18:12,304 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:12,304 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:12,306 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=bf7ade1d680d2bf0f19c877d69800cad, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:12,316 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf7ade1d680d2bf0f19c877d69800cad, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:12,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-05T09:18:12,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure bf7ade1d680d2bf0f19c877d69800cad, server=33186fd147fa,43109,1733390206335 in 234 msec 2024-12-05T09:18:12,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T09:18:12,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bf7ade1d680d2bf0f19c877d69800cad, ASSIGN in 411 msec 2024-12-05T09:18:12,354 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:12,355 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390292355"}]},"ts":"1733390292355"} 2024-12-05T09:18:12,367 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-05T09:18:12,371 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:12,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 638 msec 2024-12-05T09:18:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T09:18:12,757 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-05T09:18:12,757 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-05T09:18:12,757 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:12,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-05T09:18:12,765 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:12,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-05T09:18:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:12,790 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:12,790 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-05T09:18:12,804 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-05T09:18:12,809 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-05T09:18:12,811 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-05T09:18:12,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3212928119170138087 2024-12-05T09:18:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390295811, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:18:12,817 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-05T09:18:12,819 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-05T09:18:12,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:18:12,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:18:12,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3212928119170138085 2024-12-05T09:18:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390295823, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:18:12,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:18:12,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3212928119170138084 2024-12-05T09:18:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390295830, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:18:12,836 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:18:12,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3212928119170138083 2024-12-05T09:18:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:60902 deadline: 1733390295836, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T09:18:12,847 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T09:18:12,859 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-707245712_22 at /127.0.0.1:34044 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=290 (was 202) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5994 (was 6037) 2024-12-05T09:18:12,872 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=290, ProcessCount=11, AvailableMemoryMB=5993 2024-12-05T09:18:12,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-05T09:18:12,881 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:12,881 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:12,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-05T09:18:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-05T09:18:12,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:12,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741843_1019 (size=418) 2024-12-05T09:18:13,327 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 317f74f9f84d384624caa5a8c8d7c372, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:13,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741844_1020 (size=69) 2024-12-05T09:18:13,356 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:13,356 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 317f74f9f84d384624caa5a8c8d7c372, disabling compactions & flushes 2024-12-05T09:18:13,356 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,356 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,356 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. after waiting 0 ms 2024-12-05T09:18:13,356 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,356 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,356 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 317f74f9f84d384624caa5a8c8d7c372: Waiting for close lock at 1733390293356Disabling compacts and flushes for region at 1733390293356Disabling writes for close at 1733390293356Writing region close event to WAL at 1733390293356Closed at 1733390293356 2024-12-05T09:18:13,361 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:13,361 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733390293361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390293361"}]},"ts":"1733390293361"} 2024-12-05T09:18:13,365 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:13,367 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:13,367 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390293367"}]},"ts":"1733390293367"} 2024-12-05T09:18:13,370 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-05T09:18:13,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=317f74f9f84d384624caa5a8c8d7c372, ASSIGN}] 2024-12-05T09:18:13,373 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=317f74f9f84d384624caa5a8c8d7c372, ASSIGN 2024-12-05T09:18:13,375 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=317f74f9f84d384624caa5a8c8d7c372, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:13,420 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-05T09:18:13,527 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=317f74f9f84d384624caa5a8c8d7c372, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:13,532 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=317f74f9f84d384624caa5a8c8d7c372, ASSIGN because future has completed 2024-12-05T09:18:13,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 317f74f9f84d384624caa5a8c8d7c372, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:13,695 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,695 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 317f74f9f84d384624caa5a8c8d7c372, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:13,696 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,696 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:13,696 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,696 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,703 INFO [StoreOpener-317f74f9f84d384624caa5a8c8d7c372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,716 INFO [StoreOpener-317f74f9f84d384624caa5a8c8d7c372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 317f74f9f84d384624caa5a8c8d7c372 columnFamilyName testFamily 2024-12-05T09:18:13,716 DEBUG [StoreOpener-317f74f9f84d384624caa5a8c8d7c372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:13,719 INFO [StoreOpener-317f74f9f84d384624caa5a8c8d7c372-1 {}] regionserver.HStore(327): Store=317f74f9f84d384624caa5a8c8d7c372/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:13,720 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,723 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,724 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,725 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,726 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,741 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,746 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:13,747 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 317f74f9f84d384624caa5a8c8d7c372; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70084206, jitterRate=0.04433605074882507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:13,748 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:13,748 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 317f74f9f84d384624caa5a8c8d7c372: Running coprocessor pre-open hook at 1733390293697Writing region info on filesystem at 1733390293697Initializing all the Stores at 1733390293699 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390293699Cleaning up temporary data from old regions at 1733390293726 (+27 ms)Running coprocessor post-open hooks at 1733390293748 (+22 ms)Region opened successfully at 1733390293748 2024-12-05T09:18:13,751 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., pid=18, masterSystemTime=1733390293687 2024-12-05T09:18:13,756 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=317f74f9f84d384624caa5a8c8d7c372, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:13,756 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,757 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:13,764 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 317f74f9f84d384624caa5a8c8d7c372, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:13,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T09:18:13,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 317f74f9f84d384624caa5a8c8d7c372, server=33186fd147fa,43109,1733390206335 in 235 msec 2024-12-05T09:18:13,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-05T09:18:13,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=317f74f9f84d384624caa5a8c8d7c372, ASSIGN in 405 msec 2024-12-05T09:18:13,805 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:13,806 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390293805"}]},"ts":"1733390293805"} 2024-12-05T09:18:13,812 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-05T09:18:13,816 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:13,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 940 msec 2024-12-05T09:18:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-05T09:18:13,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-05T09:18:13,896 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-05T09:18:13,897 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:13,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-05T09:18:13,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:13,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-05T09:18:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:13,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:13,951 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-05T09:18:13,956 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T09:18:13,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T09:18:14,804 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T09:18:14,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=33186fd147fa:43109 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-05T09:18:14,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-05T09:18:14,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2 from cache 2024-12-05T09:18:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-05T09:18:15,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-05T09:18:15,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:15,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-05T09:18:15,961 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:15,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T09:18:16,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=33186fd147fa:43109 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=991ms, rpcTimeout=987ms 2024-12-05T09:18:16,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=991ms, rpcTimeout=987ms 2024-12-05T09:18:16,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., hostname=33186fd147fa,43109,1733390206335, seqNum=2 from cache 2024-12-05T09:18:16,975 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 223), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=290 (was 290), ProcessCount=11 (was 11), AvailableMemoryMB=5665 (was 5993) 2024-12-05T09:18:16,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=290, ProcessCount=11, AvailableMemoryMB=5664 2024-12-05T09:18:16,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-05T09:18:16,993 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:16,993 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:16,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-05T09:18:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T09:18:16,996 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:17,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741845_1021 (size=416) 2024-12-05T09:18:17,041 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b1d6ebd44e95ceab1ce369bb3a6f3c81, NAME => 'testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741846_1022 (size=67) 2024-12-05T09:18:17,457 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:17,457 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing b1d6ebd44e95ceab1ce369bb3a6f3c81, disabling compactions & flushes 2024-12-05T09:18:17,457 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,457 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,458 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. after waiting 0 ms 2024-12-05T09:18:17,458 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,458 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,458 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for b1d6ebd44e95ceab1ce369bb3a6f3c81: Waiting for close lock at 1733390297457Disabling compacts and flushes for region at 1733390297457Disabling writes for close at 1733390297458 (+1 ms)Writing region close event to WAL at 1733390297458Closed at 1733390297458 2024-12-05T09:18:17,460 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:17,461 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733390297461"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390297461"}]},"ts":"1733390297461"} 2024-12-05T09:18:17,465 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:17,468 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:17,468 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390297468"}]},"ts":"1733390297468"} 2024-12-05T09:18:17,471 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-05T09:18:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-05T09:18:17,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=b1d6ebd44e95ceab1ce369bb3a6f3c81, ASSIGN}] 2024-12-05T09:18:17,474 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=b1d6ebd44e95ceab1ce369bb3a6f3c81, ASSIGN 2024-12-05T09:18:17,479 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=b1d6ebd44e95ceab1ce369bb3a6f3c81, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:17,630 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b1d6ebd44e95ceab1ce369bb3a6f3c81, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:17,633 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=b1d6ebd44e95ceab1ce369bb3a6f3c81, ASSIGN because future has completed 2024-12-05T09:18:17,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1d6ebd44e95ceab1ce369bb3a6f3c81, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:17,754 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:18:17,794 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,794 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => b1d6ebd44e95ceab1ce369bb3a6f3c81, NAME => 'testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:17,795 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,795 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:17,795 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,795 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,798 INFO [StoreOpener-b1d6ebd44e95ceab1ce369bb3a6f3c81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,803 INFO [StoreOpener-b1d6ebd44e95ceab1ce369bb3a6f3c81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1d6ebd44e95ceab1ce369bb3a6f3c81 columnFamilyName testFamily 2024-12-05T09:18:17,803 DEBUG [StoreOpener-b1d6ebd44e95ceab1ce369bb3a6f3c81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:17,804 INFO [StoreOpener-b1d6ebd44e95ceab1ce369bb3a6f3c81-1 {}] regionserver.HStore(327): Store=b1d6ebd44e95ceab1ce369bb3a6f3c81/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:17,804 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,811 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,812 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,813 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,813 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,816 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,822 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:17,822 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened b1d6ebd44e95ceab1ce369bb3a6f3c81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68456454, jitterRate=0.020080655813217163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:17,823 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:17,824 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for b1d6ebd44e95ceab1ce369bb3a6f3c81: Running coprocessor pre-open hook at 1733390297795Writing region info on filesystem at 1733390297795Initializing all the Stores at 1733390297797 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390297797Cleaning up temporary data from old regions at 1733390297813 (+16 ms)Running coprocessor post-open hooks at 1733390297823 (+10 ms)Region opened successfully at 1733390297823 2024-12-05T09:18:17,827 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81., pid=21, masterSystemTime=1733390297787 2024-12-05T09:18:17,832 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,832 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:17,833 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b1d6ebd44e95ceab1ce369bb3a6f3c81, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:17,837 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1d6ebd44e95ceab1ce369bb3a6f3c81, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:17,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-05T09:18:17,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure b1d6ebd44e95ceab1ce369bb3a6f3c81, server=33186fd147fa,43109,1733390206335 in 205 msec 2024-12-05T09:18:17,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T09:18:17,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=b1d6ebd44e95ceab1ce369bb3a6f3c81, ASSIGN in 374 msec 2024-12-05T09:18:17,853 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:17,854 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390297854"}]},"ts":"1733390297854"} 2024-12-05T09:18:17,857 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-05T09:18:17,859 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:17,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 871 msec 2024-12-05T09:18:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T09:18:18,016 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-05T09:18:18,016 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-05T09:18:18,016 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:18,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-05T09:18:18,023 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:18,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-05T09:18:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:18,043 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:18,043 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-05T09:18:18,047 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T09:18:18,051 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T09:18:18,053 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T09:18:18,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-05T09:18:19,422 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-05T09:18:19,422 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-05T09:18:21,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138072' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T09:18:21,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:60902 deadline: 1733390302060, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138072' does not exist 2024-12-05T09:18:21,064 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 33186fd147fa,43109,1733390206335 for closing scanner id = -3212928119170138072 for b1d6ebd44e95ceab1ce369bb3a6f3c81 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138072' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T09:18:21,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 223), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=331 (was 290) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5616 (was 5664) 2024-12-05T09:18:21,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=331, ProcessCount=11, AvailableMemoryMB=5616 2024-12-05T09:18:21,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-05T09:18:21,084 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:21,084 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:21,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-05T09:18:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-05T09:18:21,087 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:21,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741847_1023 (size=411) 2024-12-05T09:18:21,105 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 803195719f1010b0f16409c34c03b6b2, NAME => 'testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741848_1024 (size=62) 2024-12-05T09:18:21,518 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:21,518 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 803195719f1010b0f16409c34c03b6b2, disabling compactions & flushes 2024-12-05T09:18:21,519 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,519 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,519 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. after waiting 0 ms 2024-12-05T09:18:21,519 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,519 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,519 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 803195719f1010b0f16409c34c03b6b2: Waiting for close lock at 1733390301518Disabling compacts and flushes for region at 1733390301518Disabling writes for close at 1733390301519 (+1 ms)Writing region close event to WAL at 1733390301519Closed at 1733390301519 2024-12-05T09:18:21,522 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:21,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733390301522"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390301522"}]},"ts":"1733390301522"} 2024-12-05T09:18:21,527 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:21,531 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:21,531 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390301531"}]},"ts":"1733390301531"} 2024-12-05T09:18:21,535 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-05T09:18:21,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=803195719f1010b0f16409c34c03b6b2, ASSIGN}] 2024-12-05T09:18:21,538 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=803195719f1010b0f16409c34c03b6b2, ASSIGN 2024-12-05T09:18:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-05T09:18:21,558 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=803195719f1010b0f16409c34c03b6b2, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:21,710 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=803195719f1010b0f16409c34c03b6b2, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:21,717 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=803195719f1010b0f16409c34c03b6b2, ASSIGN because future has completed 2024-12-05T09:18:21,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 803195719f1010b0f16409c34c03b6b2, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:21,877 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,877 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 803195719f1010b0f16409c34c03b6b2, NAME => 'testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:21,877 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,877 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:21,877 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,878 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,880 INFO [StoreOpener-803195719f1010b0f16409c34c03b6b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,882 INFO [StoreOpener-803195719f1010b0f16409c34c03b6b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 803195719f1010b0f16409c34c03b6b2 columnFamilyName testFamily 2024-12-05T09:18:21,882 DEBUG [StoreOpener-803195719f1010b0f16409c34c03b6b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:21,883 INFO [StoreOpener-803195719f1010b0f16409c34c03b6b2-1 {}] regionserver.HStore(327): Store=803195719f1010b0f16409c34c03b6b2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:21,883 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,884 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,885 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,885 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,885 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,888 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,891 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:21,894 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 803195719f1010b0f16409c34c03b6b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59883401, jitterRate=-0.10766778886318207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:21,894 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:21,895 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 803195719f1010b0f16409c34c03b6b2: Running coprocessor pre-open hook at 1733390301878Writing region info on filesystem at 1733390301878Initializing all the Stores at 1733390301879 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390301879Cleaning up temporary data from old regions at 1733390301885 (+6 ms)Running coprocessor post-open hooks at 1733390301894 (+9 ms)Region opened successfully at 1733390301895 (+1 ms) 2024-12-05T09:18:21,896 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2., pid=24, masterSystemTime=1733390301872 2024-12-05T09:18:21,901 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,901 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:21,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=803195719f1010b0f16409c34c03b6b2, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:21,904 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 803195719f1010b0f16409c34c03b6b2, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:21,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T09:18:21,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 803195719f1010b0f16409c34c03b6b2, server=33186fd147fa,43109,1733390206335 in 189 msec 2024-12-05T09:18:21,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-05T09:18:21,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=803195719f1010b0f16409c34c03b6b2, ASSIGN in 374 msec 2024-12-05T09:18:21,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:21,914 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390301914"}]},"ts":"1733390301914"} 2024-12-05T09:18:21,917 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-05T09:18:21,919 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:21,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 839 msec 2024-12-05T09:18:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-05T09:18:22,096 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-05T09:18:22,096 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-05T09:18:22,096 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:22,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-05T09:18:22,102 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:22,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-05T09:18:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:22,117 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:22,117 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-05T09:18:22,121 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T09:18:22,124 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T09:18:22,125 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T09:18:22,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-05T09:18:23,297 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:18:25,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138068' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T09:18:25,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:60902 deadline: 1733390306130, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138068' does not exist 2024-12-05T09:18:25,134 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 33186fd147fa,43109,1733390206335 for closing scanner id = -3212928119170138068 for 803195719f1010b0f16409c34c03b6b2 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138068' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T09:18:25,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 223), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 331) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5562 (was 5616) 2024-12-05T09:18:25,150 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=337, ProcessCount=11, AvailableMemoryMB=5561 2024-12-05T09:18:25,153 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:25,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-05T09:18:25,157 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:25,157 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:25,158 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-05T09:18:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T09:18:25,160 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:25,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741849_1025 (size=419) 2024-12-05T09:18:25,190 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 60629aa05d3ff2d9cf1d01473e598ae2, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:25,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741850_1026 (size=70) 2024-12-05T09:18:25,423 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-05T09:18:25,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:25,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 60629aa05d3ff2d9cf1d01473e598ae2, disabling compactions & flushes 2024-12-05T09:18:25,611 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. after waiting 0 ms 2024-12-05T09:18:25,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,611 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 60629aa05d3ff2d9cf1d01473e598ae2: Waiting for close lock at 1733390305611Disabling compacts and flushes for region at 1733390305611Disabling writes for close at 1733390305611Writing region close event to WAL at 1733390305611Closed at 1733390305611 2024-12-05T09:18:25,613 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:25,614 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733390305613"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390305613"}]},"ts":"1733390305613"} 2024-12-05T09:18:25,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:25,618 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:25,619 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390305618"}]},"ts":"1733390305618"} 2024-12-05T09:18:25,623 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-05T09:18:25,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=60629aa05d3ff2d9cf1d01473e598ae2, ASSIGN}] 2024-12-05T09:18:25,625 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=60629aa05d3ff2d9cf1d01473e598ae2, ASSIGN 2024-12-05T09:18:25,626 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=60629aa05d3ff2d9cf1d01473e598ae2, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-05T09:18:25,778 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=60629aa05d3ff2d9cf1d01473e598ae2, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:25,783 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=60629aa05d3ff2d9cf1d01473e598ae2, ASSIGN because future has completed 2024-12-05T09:18:25,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60629aa05d3ff2d9cf1d01473e598ae2, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:25,942 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 60629aa05d3ff2d9cf1d01473e598ae2, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:25,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:25,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,951 INFO [StoreOpener-60629aa05d3ff2d9cf1d01473e598ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,954 INFO [StoreOpener-60629aa05d3ff2d9cf1d01473e598ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60629aa05d3ff2d9cf1d01473e598ae2 columnFamilyName testFamily 2024-12-05T09:18:25,954 DEBUG [StoreOpener-60629aa05d3ff2d9cf1d01473e598ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:25,956 INFO [StoreOpener-60629aa05d3ff2d9cf1d01473e598ae2-1 {}] regionserver.HStore(327): Store=60629aa05d3ff2d9cf1d01473e598ae2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:25,957 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,958 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-05T09:18:25,959 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:25,960 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,960 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-05T09:18:25,960 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:25,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:25,969 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 60629aa05d3ff2d9cf1d01473e598ae2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68586575, jitterRate=0.022019609808921814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:25,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:25,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 60629aa05d3ff2d9cf1d01473e598ae2: Running coprocessor pre-open hook at 1733390305944Writing region info on filesystem at 1733390305944Initializing all the Stores at 1733390305950 (+6 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390305951 (+1 ms)Cleaning up temporary data from old regions at 1733390305960 (+9 ms)Running coprocessor post-open hooks at 1733390305969 (+9 ms)Region opened successfully at 1733390305969 2024-12-05T09:18:25,971 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., pid=27, masterSystemTime=1733390305936 2024-12-05T09:18:25,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,978 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:25,979 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=60629aa05d3ff2d9cf1d01473e598ae2, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:25,982 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60629aa05d3ff2d9cf1d01473e598ae2, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:25,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-05T09:18:25,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 60629aa05d3ff2d9cf1d01473e598ae2, server=33186fd147fa,43109,1733390206335 in 204 msec 2024-12-05T09:18:25,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T09:18:25,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=60629aa05d3ff2d9cf1d01473e598ae2, ASSIGN in 369 msec 2024-12-05T09:18:25,997 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:25,997 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390305997"}]},"ts":"1733390305997"} 2024-12-05T09:18:26,000 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-05T09:18:26,002 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:26,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 849 msec 2024-12-05T09:18:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T09:18:26,166 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-05T09:18:26,166 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-05T09:18:26,166 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:26,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-05T09:18:26,173 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:26,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-05T09:18:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:26,192 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:26,192 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-05T09:18:26,197 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T09:18:26,199 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-05T09:18:30,640 INFO [master/33186fd147fa:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T09:18:30,640 INFO [master/33186fd147fa:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T09:18:31,447 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:18:31,473 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-05T09:18:32,200 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., hostname=33186fd147fa,43109,1733390206335, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., hostname=33186fd147fa,43109,1733390206335, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=33186fd147fa:43109 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-12-05T09:18:32,201 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., hostname=33186fd147fa,43109,1733390206335, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-12-05T09:18:32,201 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., hostname=33186fd147fa,43109,1733390206335, seqNum=2 from cache 2024-12-05T09:18:32,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 337), ProcessCount=11 (was 11), AvailableMemoryMB=5535 (was 5561) 2024-12-05T09:18:32,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=329, ProcessCount=11, AvailableMemoryMB=5534 2024-12-05T09:18:32,227 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:32,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-05T09:18:32,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:32,231 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:32,231 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-05T09:18:32,233 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T09:18:32,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741851_1027 (size=407) 2024-12-05T09:18:32,647 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 697a907d3b2882bd9e7312b0f299cff9, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:32,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741852_1028 (size=58) 2024-12-05T09:18:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-05T09:18:33,060 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:33,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 697a907d3b2882bd9e7312b0f299cff9, disabling compactions & flushes 2024-12-05T09:18:33,061 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. after waiting 0 ms 2024-12-05T09:18:33,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,061 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 697a907d3b2882bd9e7312b0f299cff9: Waiting for close lock at 1733390313061Disabling compacts and flushes for region at 1733390313061Disabling writes for close at 1733390313061Writing region close event to WAL at 1733390313061Closed at 1733390313061 2024-12-05T09:18:33,064 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:33,064 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733390313064"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390313064"}]},"ts":"1733390313064"} 2024-12-05T09:18:33,068 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:33,071 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:33,071 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390313071"}]},"ts":"1733390313071"} 2024-12-05T09:18:33,074 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-05T09:18:33,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=697a907d3b2882bd9e7312b0f299cff9, ASSIGN}] 2024-12-05T09:18:33,077 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=697a907d3b2882bd9e7312b0f299cff9, ASSIGN 2024-12-05T09:18:33,078 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=697a907d3b2882bd9e7312b0f299cff9, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:33,230 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=697a907d3b2882bd9e7312b0f299cff9, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:33,236 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=697a907d3b2882bd9e7312b0f299cff9, ASSIGN because future has completed 2024-12-05T09:18:33,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 697a907d3b2882bd9e7312b0f299cff9, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T09:18:33,404 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,404 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 697a907d3b2882bd9e7312b0f299cff9, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:33,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:33,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,408 INFO [StoreOpener-697a907d3b2882bd9e7312b0f299cff9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,410 INFO [StoreOpener-697a907d3b2882bd9e7312b0f299cff9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 697a907d3b2882bd9e7312b0f299cff9 columnFamilyName testFamily 2024-12-05T09:18:33,410 DEBUG [StoreOpener-697a907d3b2882bd9e7312b0f299cff9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:33,411 INFO [StoreOpener-697a907d3b2882bd9e7312b0f299cff9-1 {}] regionserver.HStore(327): Store=697a907d3b2882bd9e7312b0f299cff9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:33,411 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,412 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,412 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,413 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,413 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,417 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:33,418 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 697a907d3b2882bd9e7312b0f299cff9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62692669, jitterRate=-0.06580643355846405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:33,418 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:33,419 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 697a907d3b2882bd9e7312b0f299cff9: Running coprocessor pre-open hook at 1733390313406Writing region info on filesystem at 1733390313406Initializing all the Stores at 1733390313408 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390313408Cleaning up temporary data from old regions at 1733390313413 (+5 ms)Running coprocessor post-open hooks at 1733390313418 (+5 ms)Region opened successfully at 1733390313419 (+1 ms) 2024-12-05T09:18:33,420 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9., pid=30, masterSystemTime=1733390313392 2024-12-05T09:18:33,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,424 INFO [RS_OPEN_PRIORITY_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:33,424 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=697a907d3b2882bd9e7312b0f299cff9, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:33,427 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 697a907d3b2882bd9e7312b0f299cff9, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:33,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T09:18:33,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 697a907d3b2882bd9e7312b0f299cff9, server=33186fd147fa,43109,1733390206335 in 192 msec 2024-12-05T09:18:33,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-05T09:18:33,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=697a907d3b2882bd9e7312b0f299cff9, ASSIGN in 357 msec 2024-12-05T09:18:33,435 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:33,435 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390313435"}]},"ts":"1733390313435"} 2024-12-05T09:18:33,437 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-05T09:18:33,439 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:33,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2120 sec 2024-12-05T09:18:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T09:18:35,265 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-05T09:18:35,265 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-05T09:18:35,265 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:35,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-05T09:18:35,270 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:35,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-05T09:18:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:35,286 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:35,286 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-05T09:18:35,289 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T09:18:35,292 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T09:18:35,293 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T09:18:35,294 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-05T09:18:35,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-05T09:18:35,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:35,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-05T09:18:35,961 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:37,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-05T09:18:38,908 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:18:41,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138060' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T09:18:41,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43109 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:60902 deadline: 1733390327301, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138060' does not exist 2024-12-05T09:18:41,304 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 33186fd147fa,43109,1733390206335 for closing scanner id = -3212928119170138060 for 697a907d3b2882bd9e7312b0f299cff9 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3212928119170138060' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T09:18:41,311 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 329), ProcessCount=11 (was 11), AvailableMemoryMB=5453 (was 5534) 2024-12-05T09:18:41,320 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=5453 2024-12-05T09:18:41,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T09:18:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-05T09:18:41,328 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T09:18:41,328 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:41,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-05T09:18:41,330 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T09:18:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T09:18:41,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741853_1029 (size=423) 2024-12-05T09:18:41,747 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2da2075f3501c88ba29f3b026e8046c3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c 2024-12-05T09:18:41,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741854_1030 (size=74) 2024-12-05T09:18:41,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:41,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2da2075f3501c88ba29f3b026e8046c3, disabling compactions & flushes 2024-12-05T09:18:41,773 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:41,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:41,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. after waiting 0 ms 2024-12-05T09:18:41,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:41,773 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:41,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2da2075f3501c88ba29f3b026e8046c3: Waiting for close lock at 1733390321773Disabling compacts and flushes for region at 1733390321773Disabling writes for close at 1733390321773Writing region close event to WAL at 1733390321773Closed at 1733390321773 2024-12-05T09:18:41,778 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T09:18:41,779 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733390321778"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733390321778"}]},"ts":"1733390321778"} 2024-12-05T09:18:41,782 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T09:18:41,784 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T09:18:41,784 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390321784"}]},"ts":"1733390321784"} 2024-12-05T09:18:41,787 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-05T09:18:41,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2da2075f3501c88ba29f3b026e8046c3, ASSIGN}] 2024-12-05T09:18:41,790 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2da2075f3501c88ba29f3b026e8046c3, ASSIGN 2024-12-05T09:18:41,791 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2da2075f3501c88ba29f3b026e8046c3, ASSIGN; state=OFFLINE, location=33186fd147fa,43109,1733390206335; forceNewPlan=false, retain=false 2024-12-05T09:18:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-05T09:18:41,943 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=2da2075f3501c88ba29f3b026e8046c3, regionState=OPENING, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:41,947 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2da2075f3501c88ba29f3b026e8046c3, ASSIGN because future has completed 2024-12-05T09:18:41,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2da2075f3501c88ba29f3b026e8046c3, server=33186fd147fa,43109,1733390206335}] 2024-12-05T09:18:42,108 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:42,108 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 2da2075f3501c88ba29f3b026e8046c3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3.', STARTKEY => '', ENDKEY => ''} 2024-12-05T09:18:42,109 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,109 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T09:18:42,109 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,109 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,112 INFO [StoreOpener-2da2075f3501c88ba29f3b026e8046c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,115 INFO [StoreOpener-2da2075f3501c88ba29f3b026e8046c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da2075f3501c88ba29f3b026e8046c3 columnFamilyName testFamily 2024-12-05T09:18:42,116 DEBUG [StoreOpener-2da2075f3501c88ba29f3b026e8046c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T09:18:42,117 INFO [StoreOpener-2da2075f3501c88ba29f3b026e8046c3-1 {}] regionserver.HStore(327): Store=2da2075f3501c88ba29f3b026e8046c3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T09:18:42,117 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,118 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,119 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,120 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,120 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,122 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,126 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T09:18:42,127 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 2da2075f3501c88ba29f3b026e8046c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67504632, jitterRate=0.005897402763366699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T09:18:42,128 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:42,129 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 2da2075f3501c88ba29f3b026e8046c3: Running coprocessor pre-open hook at 1733390322109Writing region info on filesystem at 1733390322109Initializing all the Stores at 1733390322111 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733390322112 (+1 ms)Cleaning up temporary data from old regions at 1733390322120 (+8 ms)Running coprocessor post-open hooks at 1733390322128 (+8 ms)Region opened successfully at 1733390322129 (+1 ms) 2024-12-05T09:18:42,131 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., pid=33, masterSystemTime=1733390322101 2024-12-05T09:18:42,134 DEBUG [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:42,134 INFO [RS_OPEN_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:42,135 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=2da2075f3501c88ba29f3b026e8046c3, regionState=OPEN, openSeqNum=2, regionLocation=33186fd147fa,43109,1733390206335 2024-12-05T09:18:42,139 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2da2075f3501c88ba29f3b026e8046c3, server=33186fd147fa,43109,1733390206335 because future has completed 2024-12-05T09:18:42,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-05T09:18:42,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 2da2075f3501c88ba29f3b026e8046c3, server=33186fd147fa,43109,1733390206335 in 193 msec 2024-12-05T09:18:42,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T09:18:42,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2da2075f3501c88ba29f3b026e8046c3, ASSIGN in 357 msec 2024-12-05T09:18:42,150 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T09:18:42,150 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733390322150"}]},"ts":"1733390322150"} 2024-12-05T09:18:42,154 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-05T09:18:42,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T09:18:42,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 833 msec 2024-12-05T09:18:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T09:18:42,346 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-05T09:18:42,346 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-05T09:18:42,346 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:42,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-05T09:18:42,351 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T09:18:42,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-05T09:18:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:42,369 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T09:18:42,370 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-05T09:18:42,374 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T09:18:42,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T09:18:43,380 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=33186fd147fa:43109 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-05T09:18:43,380 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-05T09:18:43,380 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2 from cache 2024-12-05T09:18:43,478 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-05T09:18:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-05T09:18:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2] 2024-12-05T09:18:44,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43109 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T09:18:44,804 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T09:18:45,380 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=33186fd147fa:43109 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=990ms, rpcTimeout=985ms 2024-12-05T09:18:45,380 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=990ms, rpcTimeout=985ms 2024-12-05T09:18:45,380 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., hostname=33186fd147fa,43109,1733390206335, seqNum=2 from cache 2024-12-05T09:18:45,390 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 303) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5451 (was 5453) 2024-12-05T09:18:45,390 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T09:18:45,391 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T09:18:45,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:45,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:45,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T09:18:45,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T09:18:45,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T09:18:45,396 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T09:18:45,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:45,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T09:18:45,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:45,397 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T09:18:45,397 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=628880251, stopped=false 2024-12-05T09:18:45,398 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=33186fd147fa,36919,1733390205638 2024-12-05T09:18:45,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T09:18:45,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:18:45,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T09:18:45,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:18:45,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T09:18:45,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T09:18:45,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T09:18:45,453 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T09:18:45,453 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T09:18:45,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:45,454 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '33186fd147fa,43109,1733390206335' ***** 2024-12-05T09:18:45,454 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T09:18:45,454 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T09:18:45,454 INFO [RS:0;33186fd147fa:43109 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T09:18:45,454 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 317f74f9f84d384624caa5a8c8d7c372 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 75b57c100509b5527163021652761449 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 697a907d3b2882bd9e7312b0f299cff9 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 803195719f1010b0f16409c34c03b6b2 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for b1d6ebd44e95ceab1ce369bb3a6f3c81 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 2da2075f3501c88ba29f3b026e8046c3 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 49bbaa611533446894035c688795e4c6 2024-12-05T09:18:45,455 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for bf7ade1d680d2bf0f19c877d69800cad 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:45,456 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 317f74f9f84d384624caa5a8c8d7c372, disabling compactions & flushes 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(959): stopping server 33186fd147fa,43109,1733390206335 2024-12-05T09:18:45,456 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T09:18:45,456 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:45,456 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. after waiting 0 ms 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;33186fd147fa:43109. 2024-12-05T09:18:45,456 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:45,456 DEBUG [RS:0;33186fd147fa:43109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T09:18:45,456 DEBUG [RS:0;33186fd147fa:43109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T09:18:45,456 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T09:18:45,457 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-05T09:18:45,457 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T09:18:45,457 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T09:18:45,457 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T09:18:45,457 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T09:18:45,457 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T09:18:45,458 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2da2075f3501c88ba29f3b026e8046c3, 317f74f9f84d384624caa5a8c8d7c372, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 75b57c100509b5527163021652761449, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:45,459 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-05T09:18:45,459 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 317f74f9f84d384624caa5a8c8d7c372 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:45,525 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/.tmp/testFamily/91b728eb1c254290a2ef8b313bd89dbd is 61, key is row-0/testFamily:testQualifier/1733390293915/Put/seqid=0 2024-12-05T09:18:45,531 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/info/8936a68004ee40c1b14f7d1457e495dd is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146./info:regioninfo/1733390271592/Put/seqid=0 2024-12-05T09:18:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741856_1032 (size=20564) 2024-12-05T09:18:45,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741855_1031 (size=5049) 2024-12-05T09:18:45,549 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/info/8936a68004ee40c1b14f7d1457e495dd 2024-12-05T09:18:45,624 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/ns/27c72c3418a1416b86b44b770b6fc965 is 43, key is default/ns:d/1733390270047/Put/seqid=0 2024-12-05T09:18:45,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741857_1033 (size=5153) 2024-12-05T09:18:45,635 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/ns/27c72c3418a1416b86b44b770b6fc965 2024-12-05T09:18:45,659 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2da2075f3501c88ba29f3b026e8046c3, 317f74f9f84d384624caa5a8c8d7c372, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 75b57c100509b5527163021652761449, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:45,668 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/table/e6f6155a9d5d497797984a6d7eed3539 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733390271618/Put/seqid=0 2024-12-05T09:18:45,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741858_1034 (size=6699) 2024-12-05T09:18:45,681 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/table/e6f6155a9d5d497797984a6d7eed3539 2024-12-05T09:18:45,694 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/info/8936a68004ee40c1b14f7d1457e495dd as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/info/8936a68004ee40c1b14f7d1457e495dd 2024-12-05T09:18:45,707 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/info/8936a68004ee40c1b14f7d1457e495dd, entries=100, sequenceid=56, filesize=20.1 K 2024-12-05T09:18:45,710 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/ns/27c72c3418a1416b86b44b770b6fc965 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/ns/27c72c3418a1416b86b44b770b6fc965 2024-12-05T09:18:45,725 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/ns/27c72c3418a1416b86b44b770b6fc965, entries=2, sequenceid=56, filesize=5.0 K 2024-12-05T09:18:45,728 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/.tmp/table/e6f6155a9d5d497797984a6d7eed3539 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/table/e6f6155a9d5d497797984a6d7eed3539 2024-12-05T09:18:45,743 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/table/e6f6155a9d5d497797984a6d7eed3539, entries=20, sequenceid=56, filesize=6.5 K 2024-12-05T09:18:45,752 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 288ms, sequenceid=56, compaction requested=false 2024-12-05T09:18:45,769 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-05T09:18:45,775 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T09:18:45,775 INFO [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T09:18:45,775 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733390325457Running coprocessor pre-close hooks at 1733390325457Disabling compacts and flushes for region at 1733390325457Disabling writes for close at 1733390325457Obtaining lock to block concurrent updates at 1733390325459 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733390325459Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733390325468 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733390325470 (+2 ms)Flushing 1588230740/info: creating writer at 1733390325471 (+1 ms)Flushing 1588230740/info: appending metadata at 1733390325526 (+55 ms)Flushing 1588230740/info: closing flushed file at 1733390325526Flushing 1588230740/ns: creating writer at 1733390325603 (+77 ms)Flushing 1588230740/ns: appending metadata at 1733390325623 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733390325623Flushing 1588230740/table: creating writer at 1733390325648 (+25 ms)Flushing 1588230740/table: appending metadata at 1733390325668 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733390325668Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32b6ef19: reopening flushed file at 1733390325691 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3555a7f7: reopening flushed file at 1733390325709 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62acece5: reopening flushed file at 1733390325725 (+16 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 288ms, sequenceid=56, compaction requested=false at 1733390325752 (+27 ms)Writing region close event to WAL at 1733390325762 (+10 ms)Running coprocessor post-close hooks at 1733390325773 (+11 ms)Closed at 1733390325775 (+2 ms) 2024-12-05T09:18:45,776 DEBUG [RS_CLOSE_META-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T09:18:45,859 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 317f74f9f84d384624caa5a8c8d7c372, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 75b57c100509b5527163021652761449, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:45,891 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T09:18:45,952 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/.tmp/testFamily/91b728eb1c254290a2ef8b313bd89dbd 2024-12-05T09:18:45,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-05T09:18:45,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T09:18:45,960 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T09:18:45,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T09:18:45,964 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/.tmp/testFamily/91b728eb1c254290a2ef8b313bd89dbd as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/testFamily/91b728eb1c254290a2ef8b313bd89dbd 2024-12-05T09:18:45,972 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/testFamily/91b728eb1c254290a2ef8b313bd89dbd, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:45,975 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 317f74f9f84d384624caa5a8c8d7c372 in 519ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:45,980 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScanner/317f74f9f84d384624caa5a8c8d7c372/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:45,981 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:45,981 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 317f74f9f84d384624caa5a8c8d7c372: Waiting for close lock at 1733390325455Running coprocessor pre-close hooks at 1733390325456 (+1 ms)Disabling compacts and flushes for region at 1733390325456Disabling writes for close at 1733390325456Obtaining lock to block concurrent updates at 1733390325459 (+3 ms)Preparing flush snapshotting stores in 317f74f9f84d384624caa5a8c8d7c372 at 1733390325459Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390325468 (+9 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. at 1733390325470 (+2 ms)Flushing 317f74f9f84d384624caa5a8c8d7c372/testFamily: creating writer at 1733390325471 (+1 ms)Flushing 317f74f9f84d384624caa5a8c8d7c372/testFamily: appending metadata at 1733390325519 (+48 ms)Flushing 317f74f9f84d384624caa5a8c8d7c372/testFamily: closing flushed file at 1733390325522 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f01bfc1: reopening flushed file at 1733390325962 (+440 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 317f74f9f84d384624caa5a8c8d7c372 in 519ms, sequenceid=8, compaction requested=false at 1733390325975 (+13 ms)Writing region close event to WAL at 1733390325976 (+1 ms)Running coprocessor post-close hooks at 1733390325981 (+5 ms)Closed at 1733390325981 2024-12-05T09:18:45,981 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733390292876.317f74f9f84d384624caa5a8c8d7c372. 2024-12-05T09:18:45,982 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 75b57c100509b5527163021652761449, disabling compactions & flushes 2024-12-05T09:18:45,982 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:18:45,982 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:18:45,982 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. after waiting 0 ms 2024-12-05T09:18:45,982 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:18:45,982 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 75b57c100509b5527163021652761449 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:46,005 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/.tmp/testFamily/0ce29c38ed824278a22b5f5b1fc67383 is 61, key is row-0/testFamily:testQualifier/1733390276559/Put/seqid=0 2024-12-05T09:18:46,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741859_1035 (size=5049) 2024-12-05T09:18:46,059 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 75b57c100509b5527163021652761449, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:46,230 INFO [regionserver/33186fd147fa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T09:18:46,231 INFO [regionserver/33186fd147fa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T09:18:46,260 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 75b57c100509b5527163021652761449, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:46,411 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/.tmp/testFamily/0ce29c38ed824278a22b5f5b1fc67383 2024-12-05T09:18:46,422 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/.tmp/testFamily/0ce29c38ed824278a22b5f5b1fc67383 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/testFamily/0ce29c38ed824278a22b5f5b1fc67383 2024-12-05T09:18:46,434 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/testFamily/0ce29c38ed824278a22b5f5b1fc67383, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:46,435 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 75b57c100509b5527163021652761449 in 453ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:46,441 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScanner/75b57c100509b5527163021652761449/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:46,442 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:18:46,442 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 75b57c100509b5527163021652761449: Waiting for close lock at 1733390325982Running coprocessor pre-close hooks at 1733390325982Disabling compacts and flushes for region at 1733390325982Disabling writes for close at 1733390325982Obtaining lock to block concurrent updates at 1733390325982Preparing flush snapshotting stores in 75b57c100509b5527163021652761449 at 1733390325982Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390325982Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. at 1733390325983 (+1 ms)Flushing 75b57c100509b5527163021652761449/testFamily: creating writer at 1733390325983Flushing 75b57c100509b5527163021652761449/testFamily: appending metadata at 1733390326004 (+21 ms)Flushing 75b57c100509b5527163021652761449/testFamily: closing flushed file at 1733390326004Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b338df2: reopening flushed file at 1733390326420 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 75b57c100509b5527163021652761449 in 453ms, sequenceid=8, compaction requested=false at 1733390326435 (+15 ms)Writing region close event to WAL at 1733390326436 (+1 ms)Running coprocessor post-close hooks at 1733390326442 (+6 ms)Closed at 1733390326442 2024-12-05T09:18:46,442 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733390273511.75b57c100509b5527163021652761449. 2024-12-05T09:18:46,442 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 697a907d3b2882bd9e7312b0f299cff9, disabling compactions & flushes 2024-12-05T09:18:46,442 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:46,442 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:46,442 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. after waiting 0 ms 2024-12-05T09:18:46,442 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:46,443 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 697a907d3b2882bd9e7312b0f299cff9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:46,460 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-12-05T09:18:46,460 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/.tmp/testFamily/1843e4d489a34fa9ad3c5d8467633d2b is 61, key is row-0/testFamily:testQualifier/1733390315275/Put/seqid=0 2024-12-05T09:18:46,460 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1325): Online Regions={697a907d3b2882bd9e7312b0f299cff9=hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9., 803195719f1010b0f16409c34c03b6b2=testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2., b1d6ebd44e95ceab1ce369bb3a6f3c81=testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81., 2da2075f3501c88ba29f3b026e8046c3=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., 49bbaa611533446894035c688795e4c6=hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6., cd4f09cb5a3393091a8bbb291e207146=testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146., bf7ade1d680d2bf0f19c877d69800cad=testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad., 60629aa05d3ff2d9cf1d01473e598ae2=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.} 2024-12-05T09:18:46,460 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:46,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741860_1036 (size=5049) 2024-12-05T09:18:46,661 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:46,861 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 697a907d3b2882bd9e7312b0f299cff9, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:46,869 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/.tmp/testFamily/1843e4d489a34fa9ad3c5d8467633d2b 2024-12-05T09:18:46,881 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/.tmp/testFamily/1843e4d489a34fa9ad3c5d8467633d2b as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/testFamily/1843e4d489a34fa9ad3c5d8467633d2b 2024-12-05T09:18:46,889 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/testFamily/1843e4d489a34fa9ad3c5d8467633d2b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:46,890 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 697a907d3b2882bd9e7312b0f299cff9 in 448ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:46,895 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNext/697a907d3b2882bd9e7312b0f299cff9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:46,896 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:46,896 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 697a907d3b2882bd9e7312b0f299cff9: Waiting for close lock at 1733390326442Running coprocessor pre-close hooks at 1733390326442Disabling compacts and flushes for region at 1733390326442Disabling writes for close at 1733390326442Obtaining lock to block concurrent updates at 1733390326443 (+1 ms)Preparing flush snapshotting stores in 697a907d3b2882bd9e7312b0f299cff9 at 1733390326443Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390326443Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. at 1733390326443Flushing 697a907d3b2882bd9e7312b0f299cff9/testFamily: creating writer at 1733390326444 (+1 ms)Flushing 697a907d3b2882bd9e7312b0f299cff9/testFamily: appending metadata at 1733390326460 (+16 ms)Flushing 697a907d3b2882bd9e7312b0f299cff9/testFamily: closing flushed file at 1733390326460Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28ae25ce: reopening flushed file at 1733390326880 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 697a907d3b2882bd9e7312b0f299cff9 in 448ms, sequenceid=8, compaction requested=false at 1733390326890 (+10 ms)Writing region close event to WAL at 1733390326891 (+1 ms)Running coprocessor post-close hooks at 1733390326896 (+5 ms)Closed at 1733390326896 2024-12-05T09:18:46,896 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733390312226.697a907d3b2882bd9e7312b0f299cff9. 2024-12-05T09:18:46,897 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 803195719f1010b0f16409c34c03b6b2, disabling compactions & flushes 2024-12-05T09:18:46,897 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:46,897 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:46,897 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. after waiting 0 ms 2024-12-05T09:18:46,897 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:46,897 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 803195719f1010b0f16409c34c03b6b2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:46,919 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/.tmp/testFamily/04ee22101cd541869992784c27ae8a67 is 61, key is row-0/testFamily:testQualifier/1733390302108/Put/seqid=0 2024-12-05T09:18:46,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741861_1037 (size=5049) 2024-12-05T09:18:47,061 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:47,262 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, 803195719f1010b0f16409c34c03b6b2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:47,336 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/.tmp/testFamily/04ee22101cd541869992784c27ae8a67 2024-12-05T09:18:47,346 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/.tmp/testFamily/04ee22101cd541869992784c27ae8a67 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/testFamily/04ee22101cd541869992784c27ae8a67 2024-12-05T09:18:47,354 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/testFamily/04ee22101cd541869992784c27ae8a67, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:47,356 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 803195719f1010b0f16409c34c03b6b2 in 459ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:47,361 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNext/803195719f1010b0f16409c34c03b6b2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:47,362 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:47,362 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 803195719f1010b0f16409c34c03b6b2: Waiting for close lock at 1733390326897Running coprocessor pre-close hooks at 1733390326897Disabling compacts and flushes for region at 1733390326897Disabling writes for close at 1733390326897Obtaining lock to block concurrent updates at 1733390326897Preparing flush snapshotting stores in 803195719f1010b0f16409c34c03b6b2 at 1733390326897Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390326897Flushing stores of testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. at 1733390326898 (+1 ms)Flushing 803195719f1010b0f16409c34c03b6b2/testFamily: creating writer at 1733390326898Flushing 803195719f1010b0f16409c34c03b6b2/testFamily: appending metadata at 1733390326919 (+21 ms)Flushing 803195719f1010b0f16409c34c03b6b2/testFamily: closing flushed file at 1733390326919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@416a15c0: reopening flushed file at 1733390327345 (+426 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 803195719f1010b0f16409c34c03b6b2 in 459ms, sequenceid=8, compaction requested=false at 1733390327356 (+11 ms)Writing region close event to WAL at 1733390327357 (+1 ms)Running coprocessor post-close hooks at 1733390327362 (+5 ms)Closed at 1733390327362 2024-12-05T09:18:47,362 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733390301080.803195719f1010b0f16409c34c03b6b2. 2024-12-05T09:18:47,362 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b1d6ebd44e95ceab1ce369bb3a6f3c81, disabling compactions & flushes 2024-12-05T09:18:47,362 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:47,362 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:47,362 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. after waiting 0 ms 2024-12-05T09:18:47,362 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:47,362 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b1d6ebd44e95ceab1ce369bb3a6f3c81 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:47,379 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/.tmp/testFamily/4683a5ccec36422ebd3393fb3a63516b is 61, key is row-0/testFamily:testQualifier/1733390298029/Put/seqid=0 2024-12-05T09:18:47,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741862_1038 (size=5049) 2024-12-05T09:18:47,462 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-05T09:18:47,462 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1325): Online Regions={b1d6ebd44e95ceab1ce369bb3a6f3c81=testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81., 2da2075f3501c88ba29f3b026e8046c3=testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., 49bbaa611533446894035c688795e4c6=hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6., cd4f09cb5a3393091a8bbb291e207146=testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146., bf7ade1d680d2bf0f19c877d69800cad=testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad., 60629aa05d3ff2d9cf1d01473e598ae2=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.} 2024-12-05T09:18:47,462 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:47,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T09:18:47,662 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, b1d6ebd44e95ceab1ce369bb3a6f3c81, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:47,786 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/.tmp/testFamily/4683a5ccec36422ebd3393fb3a63516b 2024-12-05T09:18:47,796 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/.tmp/testFamily/4683a5ccec36422ebd3393fb3a63516b as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/testFamily/4683a5ccec36422ebd3393fb3a63516b 2024-12-05T09:18:47,806 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/testFamily/4683a5ccec36422ebd3393fb3a63516b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:47,807 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b1d6ebd44e95ceab1ce369bb3a6f3c81 in 445ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:47,815 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnNextAsync/b1d6ebd44e95ceab1ce369bb3a6f3c81/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:47,816 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:47,816 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b1d6ebd44e95ceab1ce369bb3a6f3c81: Waiting for close lock at 1733390327362Running coprocessor pre-close hooks at 1733390327362Disabling compacts and flushes for region at 1733390327362Disabling writes for close at 1733390327362Obtaining lock to block concurrent updates at 1733390327362Preparing flush snapshotting stores in b1d6ebd44e95ceab1ce369bb3a6f3c81 at 1733390327362Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390327363 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. at 1733390327363Flushing b1d6ebd44e95ceab1ce369bb3a6f3c81/testFamily: creating writer at 1733390327364 (+1 ms)Flushing b1d6ebd44e95ceab1ce369bb3a6f3c81/testFamily: appending metadata at 1733390327379 (+15 ms)Flushing b1d6ebd44e95ceab1ce369bb3a6f3c81/testFamily: closing flushed file at 1733390327379Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c11a7dd: reopening flushed file at 1733390327794 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b1d6ebd44e95ceab1ce369bb3a6f3c81 in 445ms, sequenceid=8, compaction requested=false at 1733390327807 (+13 ms)Writing region close event to WAL at 1733390327808 (+1 ms)Running coprocessor post-close hooks at 1733390327816 (+8 ms)Closed at 1733390327816 2024-12-05T09:18:47,817 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733390296987.b1d6ebd44e95ceab1ce369bb3a6f3c81. 2024-12-05T09:18:47,817 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2da2075f3501c88ba29f3b026e8046c3, disabling compactions & flushes 2024-12-05T09:18:47,817 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:47,817 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:47,817 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. after waiting 0 ms 2024-12-05T09:18:47,817 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:47,817 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2da2075f3501c88ba29f3b026e8046c3 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:47,842 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/.tmp/testFamily/49ed1608fb26494bb4e687149cb0fa35 is 61, key is row-0/testFamily:testQualifier/1733390322358/Put/seqid=0 2024-12-05T09:18:47,863 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 2da2075f3501c88ba29f3b026e8046c3, 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:47,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741863_1039 (size=5049) 2024-12-05T09:18:47,869 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/.tmp/testFamily/49ed1608fb26494bb4e687149cb0fa35 2024-12-05T09:18:47,898 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/.tmp/testFamily/49ed1608fb26494bb4e687149cb0fa35 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/testFamily/49ed1608fb26494bb4e687149cb0fa35 2024-12-05T09:18:47,913 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/testFamily/49ed1608fb26494bb4e687149cb0fa35, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:47,919 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2da2075f3501c88ba29f3b026e8046c3 in 102ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:47,942 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testNormalScanTimeoutOnOpenScannerAsync/2da2075f3501c88ba29f3b026e8046c3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:47,943 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:47,944 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2da2075f3501c88ba29f3b026e8046c3: Waiting for close lock at 1733390327817Running coprocessor pre-close hooks at 1733390327817Disabling compacts and flushes for region at 1733390327817Disabling writes for close at 1733390327817Obtaining lock to block concurrent updates at 1733390327817Preparing flush snapshotting stores in 2da2075f3501c88ba29f3b026e8046c3 at 1733390327817Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390327817Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. at 1733390327820 (+3 ms)Flushing 2da2075f3501c88ba29f3b026e8046c3/testFamily: creating writer at 1733390327820Flushing 2da2075f3501c88ba29f3b026e8046c3/testFamily: appending metadata at 1733390327841 (+21 ms)Flushing 2da2075f3501c88ba29f3b026e8046c3/testFamily: closing flushed file at 1733390327842 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bf6ac42: reopening flushed file at 1733390327892 (+50 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2da2075f3501c88ba29f3b026e8046c3 in 102ms, sequenceid=8, compaction requested=false at 1733390327919 (+27 ms)Writing region close event to WAL at 1733390327925 (+6 ms)Running coprocessor post-close hooks at 1733390327943 (+18 ms)Closed at 1733390327943 2024-12-05T09:18:47,944 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733390321323.2da2075f3501c88ba29f3b026e8046c3. 2024-12-05T09:18:47,944 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49bbaa611533446894035c688795e4c6, disabling compactions & flushes 2024-12-05T09:18:47,944 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:47,944 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:47,944 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. after waiting 0 ms 2024-12-05T09:18:47,944 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:47,944 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 49bbaa611533446894035c688795e4c6 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:47,981 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/.tmp/testFamily/55aae67bac8e4d2c9b37d0baf6e2ed18 is 61, key is row-0/testFamily:testQualifier/1733390285665/Put/seqid=0 2024-12-05T09:18:48,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741864_1040 (size=5049) 2024-12-05T09:18:48,063 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:48,263 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 49bbaa611533446894035c688795e4c6, 60629aa05d3ff2d9cf1d01473e598ae2, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:48,404 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/.tmp/testFamily/55aae67bac8e4d2c9b37d0baf6e2ed18 2024-12-05T09:18:48,414 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/.tmp/testFamily/55aae67bac8e4d2c9b37d0baf6e2ed18 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/testFamily/55aae67bac8e4d2c9b37d0baf6e2ed18 2024-12-05T09:18:48,422 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/testFamily/55aae67bac8e4d2c9b37d0baf6e2ed18, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:48,424 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 49bbaa611533446894035c688795e4c6 in 480ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:48,428 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnNextAsync/49bbaa611533446894035c688795e4c6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:48,429 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:48,429 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49bbaa611533446894035c688795e4c6: Waiting for close lock at 1733390327944Running coprocessor pre-close hooks at 1733390327944Disabling compacts and flushes for region at 1733390327944Disabling writes for close at 1733390327944Obtaining lock to block concurrent updates at 1733390327944Preparing flush snapshotting stores in 49bbaa611533446894035c688795e4c6 at 1733390327944Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390327945 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. at 1733390327946 (+1 ms)Flushing 49bbaa611533446894035c688795e4c6/testFamily: creating writer at 1733390327946Flushing 49bbaa611533446894035c688795e4c6/testFamily: appending metadata at 1733390327980 (+34 ms)Flushing 49bbaa611533446894035c688795e4c6/testFamily: closing flushed file at 1733390327981 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71dcb964: reopening flushed file at 1733390328412 (+431 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 49bbaa611533446894035c688795e4c6 in 480ms, sequenceid=8, compaction requested=false at 1733390328424 (+12 ms)Writing region close event to WAL at 1733390328424Running coprocessor post-close hooks at 1733390328429 (+5 ms)Closed at 1733390328429 2024-12-05T09:18:48,429 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733390282606.49bbaa611533446894035c688795e4c6. 2024-12-05T09:18:48,429 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd4f09cb5a3393091a8bbb291e207146, disabling compactions & flushes 2024-12-05T09:18:48,429 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:18:48,429 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:18:48,430 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. after waiting 0 ms 2024-12-05T09:18:48,430 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:18:48,430 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cd4f09cb5a3393091a8bbb291e207146 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:48,446 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/.tmp/testFamily/1f2aa3709172444eb603aa8053da5544 is 61, key is row-0/testFamily:testQualifier/1733390273425/Put/seqid=0 2024-12-05T09:18:48,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741865_1041 (size=5049) 2024-12-05T09:18:48,464 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-05T09:18:48,464 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1325): Online Regions={cd4f09cb5a3393091a8bbb291e207146=testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146., bf7ade1d680d2bf0f19c877d69800cad=testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad., 60629aa05d3ff2d9cf1d01473e598ae2=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2.} 2024-12-05T09:18:48,464 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 60629aa05d3ff2d9cf1d01473e598ae2, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:48,664 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 60629aa05d3ff2d9cf1d01473e598ae2, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:48,851 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/.tmp/testFamily/1f2aa3709172444eb603aa8053da5544 2024-12-05T09:18:48,860 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/.tmp/testFamily/1f2aa3709172444eb603aa8053da5544 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/testFamily/1f2aa3709172444eb603aa8053da5544 2024-12-05T09:18:48,864 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 60629aa05d3ff2d9cf1d01473e598ae2, bf7ade1d680d2bf0f19c877d69800cad, cd4f09cb5a3393091a8bbb291e207146 2024-12-05T09:18:48,869 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/testFamily/1f2aa3709172444eb603aa8053da5544, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:48,870 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cd4f09cb5a3393091a8bbb291e207146 in 440ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:48,875 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/cd4f09cb5a3393091a8bbb291e207146/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:48,876 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:18:48,876 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd4f09cb5a3393091a8bbb291e207146: Waiting for close lock at 1733390328429Running coprocessor pre-close hooks at 1733390328429Disabling compacts and flushes for region at 1733390328429Disabling writes for close at 1733390328430 (+1 ms)Obtaining lock to block concurrent updates at 1733390328430Preparing flush snapshotting stores in cd4f09cb5a3393091a8bbb291e207146 at 1733390328430Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390328430Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. at 1733390328431 (+1 ms)Flushing cd4f09cb5a3393091a8bbb291e207146/testFamily: creating writer at 1733390328431Flushing cd4f09cb5a3393091a8bbb291e207146/testFamily: appending metadata at 1733390328445 (+14 ms)Flushing cd4f09cb5a3393091a8bbb291e207146/testFamily: closing flushed file at 1733390328445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bf5bc1f: reopening flushed file at 1733390328859 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cd4f09cb5a3393091a8bbb291e207146 in 440ms, sequenceid=8, compaction requested=false at 1733390328870 (+11 ms)Writing region close event to WAL at 1733390328870Running coprocessor post-close hooks at 1733390328876 (+6 ms)Closed at 1733390328876 2024-12-05T09:18:48,876 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733390270332.cd4f09cb5a3393091a8bbb291e207146. 2024-12-05T09:18:48,876 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf7ade1d680d2bf0f19c877d69800cad, disabling compactions & flushes 2024-12-05T09:18:48,877 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:48,877 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:48,877 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. after waiting 0 ms 2024-12-05T09:18:48,877 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:48,877 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bf7ade1d680d2bf0f19c877d69800cad 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:48,899 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/.tmp/testFamily/a2368fb9c9e449a0bb4678c0df5baae3 is 61, key is row-0/testFamily:testQualifier/1733390292772/Put/seqid=0 2024-12-05T09:18:48,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741866_1042 (size=5049) 2024-12-05T09:18:48,907 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/.tmp/testFamily/a2368fb9c9e449a0bb4678c0df5baae3 2024-12-05T09:18:48,919 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/.tmp/testFamily/a2368fb9c9e449a0bb4678c0df5baae3 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/testFamily/a2368fb9c9e449a0bb4678c0df5baae3 2024-12-05T09:18:48,928 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/testFamily/a2368fb9c9e449a0bb4678c0df5baae3, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:48,929 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bf7ade1d680d2bf0f19c877d69800cad in 52ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:48,934 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/default/testRetryOutOfOrderScannerNextException/bf7ade1d680d2bf0f19c877d69800cad/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:48,935 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:48,935 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf7ade1d680d2bf0f19c877d69800cad: Waiting for close lock at 1733390328876Running coprocessor pre-close hooks at 1733390328876Disabling compacts and flushes for region at 1733390328876Disabling writes for close at 1733390328877 (+1 ms)Obtaining lock to block concurrent updates at 1733390328877Preparing flush snapshotting stores in bf7ade1d680d2bf0f19c877d69800cad at 1733390328877Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390328877Flushing stores of testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. at 1733390328878 (+1 ms)Flushing bf7ade1d680d2bf0f19c877d69800cad/testFamily: creating writer at 1733390328878Flushing bf7ade1d680d2bf0f19c877d69800cad/testFamily: appending metadata at 1733390328898 (+20 ms)Flushing bf7ade1d680d2bf0f19c877d69800cad/testFamily: closing flushed file at 1733390328899 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a41825d: reopening flushed file at 1733390328918 (+19 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bf7ade1d680d2bf0f19c877d69800cad in 52ms, sequenceid=8, compaction requested=false at 1733390328929 (+11 ms)Writing region close event to WAL at 1733390328930 (+1 ms)Running coprocessor post-close hooks at 1733390328935 (+5 ms)Closed at 1733390328935 2024-12-05T09:18:48,935 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733390291732.bf7ade1d680d2bf0f19c877d69800cad. 2024-12-05T09:18:48,935 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 60629aa05d3ff2d9cf1d01473e598ae2, disabling compactions & flushes 2024-12-05T09:18:48,935 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:48,935 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:48,935 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. after waiting 0 ms 2024-12-05T09:18:48,935 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:48,935 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 60629aa05d3ff2d9cf1d01473e598ae2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T09:18:48,952 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/.tmp/testFamily/62f857cd3acb42b98c37619a35637f5f is 61, key is row-0/testFamily:testQualifier/1733390306179/Put/seqid=0 2024-12-05T09:18:48,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741867_1043 (size=5049) 2024-12-05T09:18:49,065 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:49,265 DEBUG [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1351): Waiting on 60629aa05d3ff2d9cf1d01473e598ae2 2024-12-05T09:18:49,359 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/.tmp/testFamily/62f857cd3acb42b98c37619a35637f5f 2024-12-05T09:18:49,368 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/.tmp/testFamily/62f857cd3acb42b98c37619a35637f5f as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/testFamily/62f857cd3acb42b98c37619a35637f5f 2024-12-05T09:18:49,377 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/testFamily/62f857cd3acb42b98c37619a35637f5f, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T09:18:49,378 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 60629aa05d3ff2d9cf1d01473e598ae2 in 443ms, sequenceid=8, compaction requested=false 2024-12-05T09:18:49,386 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/60629aa05d3ff2d9cf1d01473e598ae2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T09:18:49,386 INFO [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:49,387 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 60629aa05d3ff2d9cf1d01473e598ae2: Waiting for close lock at 1733390328935Running coprocessor pre-close hooks at 1733390328935Disabling compacts and flushes for region at 1733390328935Disabling writes for close at 1733390328935Obtaining lock to block concurrent updates at 1733390328936 (+1 ms)Preparing flush snapshotting stores in 60629aa05d3ff2d9cf1d01473e598ae2 at 1733390328936Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733390328936Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. at 1733390328936Flushing 60629aa05d3ff2d9cf1d01473e598ae2/testFamily: creating writer at 1733390328937 (+1 ms)Flushing 60629aa05d3ff2d9cf1d01473e598ae2/testFamily: appending metadata at 1733390328952 (+15 ms)Flushing 60629aa05d3ff2d9cf1d01473e598ae2/testFamily: closing flushed file at 1733390328952Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27e8d7ed: reopening flushed file at 1733390329367 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 60629aa05d3ff2d9cf1d01473e598ae2 in 443ms, sequenceid=8, compaction requested=false at 1733390329378 (+11 ms)Writing region close event to WAL at 1733390329379 (+1 ms)Running coprocessor post-close hooks at 1733390329386 (+7 ms)Closed at 1733390329386 2024-12-05T09:18:49,387 DEBUG [RS_CLOSE_REGION-regionserver/33186fd147fa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733390305153.60629aa05d3ff2d9cf1d01473e598ae2. 2024-12-05T09:18:49,466 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(976): stopping server 33186fd147fa,43109,1733390206335; all regions closed. 2024-12-05T09:18:49,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741834_1010 (size=19479) 2024-12-05T09:18:49,476 DEBUG [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/oldWALs 2024-12-05T09:18:49,476 INFO [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 33186fd147fa%2C43109%2C1733390206335.meta:.meta(num 1733390269812) 2024-12-05T09:18:49,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741833_1009 (size=20031) 2024-12-05T09:18:49,480 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/WALs/33186fd147fa,43109,1733390206335/33186fd147fa%2C43109%2C1733390206335.1733390269291 not finished, retry = 0 2024-12-05T09:18:49,583 DEBUG [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/oldWALs 2024-12-05T09:18:49,583 INFO [RS:0;33186fd147fa:43109 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 33186fd147fa%2C43109%2C1733390206335:(num 1733390269291) 2024-12-05T09:18:49,583 DEBUG [RS:0;33186fd147fa:43109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T09:18:49,583 INFO [RS:0;33186fd147fa:43109 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T09:18:49,583 INFO [RS:0;33186fd147fa:43109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T09:18:49,584 INFO [RS:0;33186fd147fa:43109 {}] hbase.ChoreService(370): Chore service for: regionserver/33186fd147fa:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T09:18:49,584 INFO [RS:0;33186fd147fa:43109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T09:18:49,584 INFO [regionserver/33186fd147fa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T09:18:49,585 INFO [RS:0;33186fd147fa:43109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43109 2024-12-05T09:18:49,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/33186fd147fa,43109,1733390206335 2024-12-05T09:18:49,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T09:18:49,618 INFO [RS:0;33186fd147fa:43109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T09:18:49,619 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [33186fd147fa,43109,1733390206335] 2024-12-05T09:18:49,634 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/33186fd147fa,43109,1733390206335 already deleted, retry=false 2024-12-05T09:18:49,635 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 33186fd147fa,43109,1733390206335 expired; onlineServers=0 2024-12-05T09:18:49,635 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '33186fd147fa,36919,1733390205638' ***** 2024-12-05T09:18:49,635 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T09:18:49,635 INFO [M:0;33186fd147fa:36919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T09:18:49,635 INFO [regionserver/33186fd147fa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T09:18:49,635 INFO [M:0;33186fd147fa:36919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T09:18:49,635 DEBUG [M:0;33186fd147fa:36919 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T09:18:49,636 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T09:18:49,636 DEBUG [M:0;33186fd147fa:36919 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T09:18:49,636 DEBUG [master/33186fd147fa:0:becomeActiveMaster-HFileCleaner.small.0-1733390209155 {}] cleaner.HFileCleaner(306): Exit Thread[master/33186fd147fa:0:becomeActiveMaster-HFileCleaner.small.0-1733390209155,5,FailOnTimeoutGroup] 2024-12-05T09:18:49,636 DEBUG [master/33186fd147fa:0:becomeActiveMaster-HFileCleaner.large.0-1733390209154 {}] cleaner.HFileCleaner(306): Exit Thread[master/33186fd147fa:0:becomeActiveMaster-HFileCleaner.large.0-1733390209154,5,FailOnTimeoutGroup] 2024-12-05T09:18:49,636 INFO [M:0;33186fd147fa:36919 {}] hbase.ChoreService(370): Chore service for: master/33186fd147fa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T09:18:49,636 INFO [M:0;33186fd147fa:36919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T09:18:49,636 DEBUG [M:0;33186fd147fa:36919 {}] master.HMaster(1795): Stopping service threads 2024-12-05T09:18:49,636 INFO [M:0;33186fd147fa:36919 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T09:18:49,636 INFO [M:0;33186fd147fa:36919 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T09:18:49,637 INFO [M:0;33186fd147fa:36919 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T09:18:49,637 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T09:18:49,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T09:18:49,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T09:18:49,643 DEBUG [M:0;33186fd147fa:36919 {}] zookeeper.ZKUtil(347): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T09:18:49,643 WARN [M:0;33186fd147fa:36919 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T09:18:49,644 INFO [M:0;33186fd147fa:36919 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/.lastflushedseqids 2024-12-05T09:18:49,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741868_1044 (size=460) 2024-12-05T09:18:49,657 INFO [M:0;33186fd147fa:36919 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T09:18:49,657 INFO [M:0;33186fd147fa:36919 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T09:18:49,658 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T09:18:49,658 INFO [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:18:49,658 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:18:49,658 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T09:18:49,658 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:18:49,658 INFO [M:0;33186fd147fa:36919 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.92 KB heapSize=190.38 KB 2024-12-05T09:18:49,675 DEBUG [M:0;33186fd147fa:36919 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6d3f632ab0c4710ba10160867f90ebe is 82, key is hbase:meta,,1/info:regioninfo/1733390269894/Put/seqid=0 2024-12-05T09:18:49,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741869_1045 (size=5672) 2024-12-05T09:18:49,727 INFO [RS:0;33186fd147fa:43109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T09:18:49,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T09:18:49,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43109-0x101a5f545600001, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T09:18:49,728 INFO [RS:0;33186fd147fa:43109 {}] regionserver.HRegionServer(1031): Exiting; stopping=33186fd147fa,43109,1733390206335; zookeeper connection closed. 2024-12-05T09:18:49,728 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22bfb801 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22bfb801 2024-12-05T09:18:49,729 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T09:18:50,082 INFO [M:0;33186fd147fa:36919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6d3f632ab0c4710ba10160867f90ebe 2024-12-05T09:18:50,109 DEBUG [M:0;33186fd147fa:36919 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dc47757486e747b298d378376197e2e9 is 813, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733390271624/Put/seqid=0 2024-12-05T09:18:50,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741870_1046 (size=14080) 2024-12-05T09:18:50,520 INFO [M:0;33186fd147fa:36919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.37 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dc47757486e747b298d378376197e2e9 2024-12-05T09:18:50,526 INFO [M:0;33186fd147fa:36919 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc47757486e747b298d378376197e2e9 2024-12-05T09:18:50,542 DEBUG [M:0;33186fd147fa:36919 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39a4f35fca4e42ccb50648c6a0cd95f9 is 69, key is 33186fd147fa,43109,1733390206335/rs:state/1733390239156/Put/seqid=0 2024-12-05T09:18:50,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741871_1047 (size=5156) 2024-12-05T09:18:50,950 INFO [M:0;33186fd147fa:36919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39a4f35fca4e42ccb50648c6a0cd95f9 2024-12-05T09:18:50,959 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6d3f632ab0c4710ba10160867f90ebe as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e6d3f632ab0c4710ba10160867f90ebe 2024-12-05T09:18:50,966 INFO [M:0;33186fd147fa:36919 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e6d3f632ab0c4710ba10160867f90ebe, entries=8, sequenceid=311, filesize=5.5 K 2024-12-05T09:18:50,968 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dc47757486e747b298d378376197e2e9 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dc47757486e747b298d378376197e2e9 2024-12-05T09:18:50,995 INFO [M:0;33186fd147fa:36919 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc47757486e747b298d378376197e2e9 2024-12-05T09:18:50,996 INFO [M:0;33186fd147fa:36919 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dc47757486e747b298d378376197e2e9, entries=33, sequenceid=311, filesize=13.8 K 2024-12-05T09:18:51,005 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39a4f35fca4e42ccb50648c6a0cd95f9 as hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39a4f35fca4e42ccb50648c6a0cd95f9 2024-12-05T09:18:51,019 INFO [M:0;33186fd147fa:36919 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35883/user/jenkins/test-data/2976098d-5043-2234-858b-5884f4e87f3c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39a4f35fca4e42ccb50648c6a0cd95f9, entries=1, sequenceid=311, filesize=5.0 K 2024-12-05T09:18:51,020 INFO [M:0;33186fd147fa:36919 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.92 KB/166831, heapSize ~190.09 KB/194648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1362ms, sequenceid=311, compaction requested=false 2024-12-05T09:18:51,027 INFO [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T09:18:51,027 DEBUG [M:0;33186fd147fa:36919 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733390329657Disabling compacts and flushes for region at 1733390329657Disabling writes for close at 1733390329658 (+1 ms)Obtaining lock to block concurrent updates at 1733390329658Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733390329658Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166831, getHeapSize=194888, getOffHeapSize=0, getCellsCount=358 at 1733390329658Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733390329659 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733390329659Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733390329675 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733390329675Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733390330089 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733390330108 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733390330108Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733390330527 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733390330542 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733390330542Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f6619e4: reopening flushed file at 1733390330957 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@402bbb9d: reopening flushed file at 1733390330966 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12125c60: reopening flushed file at 1733390330996 (+30 ms)Finished flush of dataSize ~162.92 KB/166831, heapSize ~190.09 KB/194648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1362ms, sequenceid=311, compaction requested=false at 1733390331020 (+24 ms)Writing region close event to WAL at 1733390331027 (+7 ms)Closed at 1733390331027 2024-12-05T09:18:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741830_1006 (size=187653) 2024-12-05T09:18:51,032 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T09:18:51,032 INFO [M:0;33186fd147fa:36919 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T09:18:51,032 INFO [M:0;33186fd147fa:36919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36919 2024-12-05T09:18:51,034 INFO [M:0;33186fd147fa:36919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T09:18:51,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T09:18:51,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36919-0x101a5f545600000, quorum=127.0.0.1:64517, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T09:18:51,168 INFO [M:0;33186fd147fa:36919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T09:18:51,174 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T09:18:51,176 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T09:18:51,177 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T09:18:51,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T09:18:51,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/hadoop.log.dir/,STOPPED} 2024-12-05T09:18:51,183 WARN [BP-164610373-172.17.0.3-1733390202473 heartbeating to localhost/127.0.0.1:35883 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T09:18:51,183 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T09:18:51,183 WARN [BP-164610373-172.17.0.3-1733390202473 heartbeating to localhost/127.0.0.1:35883 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164610373-172.17.0.3-1733390202473 (Datanode Uuid b19e8739-c778-4b27-b5c8-25196c8e86b6) service to localhost/127.0.0.1:35883 2024-12-05T09:18:51,183 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T09:18:51,185 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/data/data1/current/BP-164610373-172.17.0.3-1733390202473 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T09:18:51,185 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/cluster_8627ce20-4620-48a6-2691-18c1b06b5972/data/data2/current/BP-164610373-172.17.0.3-1733390202473 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T09:18:51,185 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T09:18:51,194 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T09:18:51,195 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T09:18:51,195 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T09:18:51,195 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T09:18:51,195 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e094e094-43ac-bbcd-bf25-803da7d11034/hadoop.log.dir/,STOPPED} 2024-12-05T09:18:51,209 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T09:18:51,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down