2024-11-08 03:10:03,383 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 03:10:03,395 main DEBUG Took 0.010625 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 03:10:03,396 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 03:10:03,396 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 03:10:03,397 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 03:10:03,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,415 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 03:10:03,426 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,428 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,428 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,429 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,430 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,430 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,431 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,431 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,432 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,433 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,433 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,434 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,435 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,435 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,436 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 03:10:03,437 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,437 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 03:10:03,438 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 03:10:03,440 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 03:10:03,441 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 03:10:03,442 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 03:10:03,443 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 03:10:03,444 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 03:10:03,452 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 03:10:03,456 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 03:10:03,458 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 03:10:03,458 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 03:10:03,459 main DEBUG createAppenders(={Console}) 2024-11-08 03:10:03,460 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-08 03:10:03,460 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 03:10:03,461 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-08 03:10:03,462 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 03:10:03,462 main DEBUG OutputStream closed 2024-11-08 03:10:03,463 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 03:10:03,463 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 03:10:03,463 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-08 03:10:03,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 03:10:03,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 03:10:03,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 03:10:03,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 03:10:03,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 03:10:03,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 03:10:03,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 03:10:03,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 03:10:03,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 03:10:03,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 03:10:03,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 03:10:03,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 03:10:03,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 03:10:03,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 03:10:03,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 03:10:03,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 03:10:03,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 03:10:03,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 03:10:03,570 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 03:10:03,571 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-08 03:10:03,571 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 03:10:03,572 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-08T03:10:03,750 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-08 03:10:03,752 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 03:10:03,752 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T03:10:03,815 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4 2024-11-08T03:10:03,844 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T03:10:03,847 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c, deleteOnExit=true 2024-11-08T03:10:03,847 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-08T03:10:03,848 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/test.cache.data in system properties and HBase conf 2024-11-08T03:10:03,848 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T03:10:03,849 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/hadoop.log.dir in system properties and HBase conf 2024-11-08T03:10:03,850 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T03:10:03,850 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T03:10:03,851 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-08T03:10:03,927 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T03:10:04,020 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T03:10:04,024 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T03:10:04,025 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T03:10:04,026 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T03:10:04,026 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T03:10:04,027 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T03:10:04,027 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T03:10:04,028 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T03:10:04,029 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T03:10:04,029 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T03:10:04,030 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/nfs.dump.dir in system properties and HBase conf 2024-11-08T03:10:04,031 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/java.io.tmpdir in system properties and HBase conf 2024-11-08T03:10:04,031 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T03:10:04,032 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T03:10:04,032 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T03:10:05,009 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T03:10:05,089 INFO [Time-limited test {}] log.Log(170): Logging initialized @2392ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T03:10:05,153 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T03:10:05,226 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T03:10:05,246 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T03:10:05,246 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T03:10:05,247 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T03:10:05,262 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T03:10:05,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/hadoop.log.dir/,AVAILABLE} 2024-11-08T03:10:05,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T03:10:05,443 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/java.io.tmpdir/jetty-localhost-42395-hadoop-hdfs-3_4_1-tests_jar-_-any-3947646915817623745/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T03:10:05,453 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:42395} 2024-11-08T03:10:05,453 INFO [Time-limited test {}] server.Server(415): Started @2757ms 2024-11-08T03:10:05,903 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T03:10:05,910 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T03:10:05,911 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T03:10:05,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T03:10:05,912 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T03:10:05,914 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/hadoop.log.dir/,AVAILABLE} 2024-11-08T03:10:05,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T03:10:06,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@183020ad{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/java.io.tmpdir/jetty-localhost-44717-hadoop-hdfs-3_4_1-tests_jar-_-any-17983045673134844630/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T03:10:06,012 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:44717} 2024-11-08T03:10:06,012 INFO [Time-limited test {}] server.Server(415): Started @3316ms 2024-11-08T03:10:06,059 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T03:10:06,833 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/dfs/data/data1/current/BP-696983924-172.17.0.2-1731035404563/current, will proceed with Du for space computation calculation, 2024-11-08T03:10:06,833 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/dfs/data/data2/current/BP-696983924-172.17.0.2-1731035404563/current, will proceed with Du for space computation calculation, 2024-11-08T03:10:06,862 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T03:10:06,904 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e078fb991b968a3 with lease ID 0x4bf15e5ba7609d19: Processing first storage report for DS-505798bb-7181-46ba-a02f-769c7cce13bc from datanode DatanodeRegistration(127.0.0.1:38891, datanodeUuid=41a4abef-dbe0-44b3-8368-6caca7824c0d, infoPort=35117, infoSecurePort=0, ipcPort=40181, storageInfo=lv=-57;cid=testClusterID;nsid=1073647470;c=1731035404563) 2024-11-08T03:10:06,905 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e078fb991b968a3 with lease ID 0x4bf15e5ba7609d19: from storage DS-505798bb-7181-46ba-a02f-769c7cce13bc node DatanodeRegistration(127.0.0.1:38891, datanodeUuid=41a4abef-dbe0-44b3-8368-6caca7824c0d, infoPort=35117, infoSecurePort=0, ipcPort=40181, storageInfo=lv=-57;cid=testClusterID;nsid=1073647470;c=1731035404563), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T03:10:06,906 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e078fb991b968a3 with lease ID 0x4bf15e5ba7609d19: Processing first storage report for DS-bf7a16dd-1227-44d1-8d95-100c3077f014 from datanode DatanodeRegistration(127.0.0.1:38891, datanodeUuid=41a4abef-dbe0-44b3-8368-6caca7824c0d, infoPort=35117, infoSecurePort=0, ipcPort=40181, storageInfo=lv=-57;cid=testClusterID;nsid=1073647470;c=1731035404563) 2024-11-08T03:10:06,906 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e078fb991b968a3 with lease ID 0x4bf15e5ba7609d19: from storage DS-bf7a16dd-1227-44d1-8d95-100c3077f014 node DatanodeRegistration(127.0.0.1:38891, datanodeUuid=41a4abef-dbe0-44b3-8368-6caca7824c0d, infoPort=35117, infoSecurePort=0, ipcPort=40181, storageInfo=lv=-57;cid=testClusterID;nsid=1073647470;c=1731035404563), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T03:10:06,939 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4 2024-11-08T03:10:07,012 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/zookeeper_0, clientPort=57399, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T03:10:07,021 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57399 2024-11-08T03:10:07,032 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:07,035 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:07,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741825_1001 (size=7) 2024-11-08T03:10:07,688 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e with version=8 2024-11-08T03:10:07,688 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/hbase-staging 2024-11-08T03:10:07,817 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T03:10:08,054 INFO [Time-limited test {}] client.ConnectionUtils(129): master/e729c2d9d3a5:0 server-side Connection retries=9 2024-11-08T03:10:08,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T03:10:08,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T03:10:08,071 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T03:10:08,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T03:10:08,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T03:10:08,191 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T03:10:08,248 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T03:10:08,262 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T03:10:08,267 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T03:10:08,301 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30797 (auto-detected) 2024-11-08T03:10:08,303 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T03:10:08,332 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:39565 2024-11-08T03:10:08,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:08,347 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:08,363 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:39565 connecting to ZooKeeper ensemble=127.0.0.1:57399 2024-11-08T03:10:08,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:395650x0, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T03:10:08,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39565-0x10119b5c5050000 connected 2024-11-08T03:10:08,513 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T03:10:08,517 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T03:10:08,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T03:10:08,526 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39565 2024-11-08T03:10:08,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39565 2024-11-08T03:10:08,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39565 2024-11-08T03:10:08,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39565 2024-11-08T03:10:08,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39565 2024-11-08T03:10:08,537 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e, hbase.cluster.distributed=false 2024-11-08T03:10:08,624 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/e729c2d9d3a5:0 server-side Connection retries=9 2024-11-08T03:10:08,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T03:10:08,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T03:10:08,625 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T03:10:08,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T03:10:08,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T03:10:08,628 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T03:10:08,631 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T03:10:08,632 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45419 2024-11-08T03:10:08,635 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T03:10:08,649 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T03:10:08,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:08,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:08,660 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45419 connecting to ZooKeeper ensemble=127.0.0.1:57399 2024-11-08T03:10:08,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454190x0, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T03:10:08,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454190x0, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T03:10:08,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454190x0, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T03:10:08,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454190x0, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T03:10:08,675 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45419-0x10119b5c5050001 connected 2024-11-08T03:10:08,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45419 2024-11-08T03:10:08,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45419 2024-11-08T03:10:08,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45419 2024-11-08T03:10:08,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45419 2024-11-08T03:10:08,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45419 2024-11-08T03:10:08,692 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:08,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T03:10:08,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T03:10:08,706 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:08,709 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e729c2d9d3a5:39565 2024-11-08T03:10:08,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T03:10:08,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T03:10:08,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:08,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:08,747 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T03:10:08,747 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T03:10:08,748 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e729c2d9d3a5,39565,1731035407811 from backup master directory 2024-11-08T03:10:08,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:08,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T03:10:08,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T03:10:08,758 WARN [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T03:10:08,759 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:08,761 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T03:10:08,762 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T03:10:08,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741826_1002 (size=42) 2024-11-08T03:10:09,231 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/hbase.id with ID: e0aad5b3-d4a8-4511-ba14-846874bcbd5e 2024-11-08T03:10:09,277 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T03:10:09,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:09,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:09,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741827_1003 (size=196) 2024-11-08T03:10:09,379 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:10:09,381 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T03:10:09,399 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:10:09,404 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T03:10:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741828_1004 (size=1189) 2024-11-08T03:10:09,861 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store 2024-11-08T03:10:09,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741829_1005 (size=34) 2024-11-08T03:10:10,285 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T03:10:10,286 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:10,289 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T03:10:10,289 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:10:10,289 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:10:10,290 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T03:10:10,290 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:10:10,290 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:10:10,290 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T03:10:10,293 WARN [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/.initializing 2024-11-08T03:10:10,293 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/WALs/e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:10,299 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T03:10:10,309 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e729c2d9d3a5%2C39565%2C1731035407811, suffix=, logDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/WALs/e729c2d9d3a5,39565,1731035407811, archiveDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/oldWALs, maxLogs=10 2024-11-08T03:10:10,327 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/WALs/e729c2d9d3a5,39565,1731035407811/e729c2d9d3a5%2C39565%2C1731035407811.1731035410313, exclude list is [], retry=0 2024-11-08T03:10:10,341 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38891,DS-505798bb-7181-46ba-a02f-769c7cce13bc,DISK] 2024-11-08T03:10:10,344 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T03:10:10,375 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/WALs/e729c2d9d3a5,39565,1731035407811/e729c2d9d3a5%2C39565%2C1731035407811.1731035410313 2024-11-08T03:10:10,376 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35117:35117)] 2024-11-08T03:10:10,376 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:10:10,377 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:10,380 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,381 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T03:10:10,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:10,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:10,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T03:10:10,446 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:10,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:10:10,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T03:10:10,451 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:10,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:10:10,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T03:10:10,455 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:10,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:10:10,461 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,462 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,471 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T03:10:10,475 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T03:10:10,480 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:10:10,481 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68294669, jitterRate=0.01766987144947052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T03:10:10,488 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T03:10:10,489 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T03:10:10,515 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723d225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:10,542 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-08T03:10:10,551 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T03:10:10,552 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T03:10:10,553 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T03:10:10,555 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T03:10:10,559 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-11-08T03:10:10,559 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T03:10:10,583 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T03:10:10,595 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T03:10:10,674 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-08T03:10:10,679 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T03:10:10,682 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T03:10:10,691 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-08T03:10:10,693 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T03:10:10,697 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T03:10:10,707 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-08T03:10:10,709 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T03:10:10,719 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T03:10:10,733 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T03:10:10,740 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T03:10:10,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T03:10:10,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T03:10:10,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:10,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:10,754 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=e729c2d9d3a5,39565,1731035407811, sessionid=0x10119b5c5050000, setting cluster-up flag (Was=false) 2024-11-08T03:10:10,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:10,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:10,807 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T03:10:10,810 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:10,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:10,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:10,858 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T03:10:10,859 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:10,905 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e729c2d9d3a5:45419 2024-11-08T03:10:10,906 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1008): ClusterId : e0aad5b3-d4a8-4511-ba14-846874bcbd5e 2024-11-08T03:10:10,909 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T03:10:10,921 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T03:10:10,921 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T03:10:10,934 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T03:10:10,934 DEBUG [RS:0;e729c2d9d3a5:45419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68112581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:10,936 DEBUG [RS:0;e729c2d9d3a5:45419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaece27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e729c2d9d3a5/172.17.0.2:0 2024-11-08T03:10:10,940 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-08T03:10:10,940 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-08T03:10:10,940 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-08T03:10:10,943 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3073): reportForDuty to master=e729c2d9d3a5,39565,1731035407811 with isa=e729c2d9d3a5/172.17.0.2:45419, startcode=1731035408623 2024-11-08T03:10:10,942 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-08T03:10:10,949 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-08T03:10:10,952 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T03:10:10,957 DEBUG [RS:0;e729c2d9d3a5:45419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T03:10:10,958 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e729c2d9d3a5,39565,1731035407811 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T03:10:10,963 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e729c2d9d3a5:0, corePoolSize=5, maxPoolSize=5 2024-11-08T03:10:10,963 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e729c2d9d3a5:0, corePoolSize=5, maxPoolSize=5 2024-11-08T03:10:10,963 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e729c2d9d3a5:0, corePoolSize=5, maxPoolSize=5 2024-11-08T03:10:10,963 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e729c2d9d3a5:0, corePoolSize=5, maxPoolSize=5 2024-11-08T03:10:10,964 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e729c2d9d3a5:0, corePoolSize=10, maxPoolSize=10 2024-11-08T03:10:10,964 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:10,964 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e729c2d9d3a5:0, corePoolSize=2, maxPoolSize=2 2024-11-08T03:10:10,964 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:10,965 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731035440965 2024-11-08T03:10:10,967 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T03:10:10,968 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T03:10:10,970 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T03:10:10,970 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-08T03:10:10,973 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T03:10:10,973 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T03:10:10,974 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T03:10:10,974 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T03:10:10,975 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:10,975 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T03:10:10,982 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:10,985 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T03:10:10,987 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T03:10:10,987 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T03:10:10,992 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T03:10:10,993 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T03:10:10,994 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e729c2d9d3a5:0:becomeActiveMaster-HFileCleaner.large.0-1731035410994,5,FailOnTimeoutGroup] 2024-11-08T03:10:10,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741831_1007 (size=1039) 2024-11-08T03:10:10,995 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e729c2d9d3a5:0:becomeActiveMaster-HFileCleaner.small.0-1731035410995,5,FailOnTimeoutGroup] 2024-11-08T03:10:10,995 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:10,995 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T03:10:10,997 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:10,997 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,001 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34249, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T03:10:11,007 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39565 {}] master.ServerManager(332): Checking decommissioned status of RegionServer e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:11,009 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39565 {}] master.ServerManager(486): Registering regionserver=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:11,022 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:10:11,023 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:36007 2024-11-08T03:10:11,023 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-08T03:10:11,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T03:10:11,033 DEBUG [RS:0;e729c2d9d3a5:45419 {}] zookeeper.ZKUtil(111): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:11,033 WARN [RS:0;e729c2d9d3a5:45419 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T03:10:11,034 INFO [RS:0;e729c2d9d3a5:45419 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T03:10:11,034 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:11,036 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e729c2d9d3a5,45419,1731035408623] 2024-11-08T03:10:11,047 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-08T03:10:11,057 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T03:10:11,073 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T03:10:11,076 INFO [RS:0;e729c2d9d3a5:45419 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T03:10:11,076 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,077 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-08T03:10:11,083 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,083 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,084 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,084 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,084 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,084 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,084 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e729c2d9d3a5:0, corePoolSize=2, maxPoolSize=2 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e729c2d9d3a5:0, corePoolSize=1, maxPoolSize=1 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e729c2d9d3a5:0, corePoolSize=3, maxPoolSize=3 2024-11-08T03:10:11,085 DEBUG [RS:0;e729c2d9d3a5:45419 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e729c2d9d3a5:0, corePoolSize=3, maxPoolSize=3 2024-11-08T03:10:11,086 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,086 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,087 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,087 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,087 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,45419,1731035408623-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T03:10:11,106 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T03:10:11,108 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,45419,1731035408623-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:11,127 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.Replication(204): e729c2d9d3a5,45419,1731035408623 started 2024-11-08T03:10:11,127 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1767): Serving as e729c2d9d3a5,45419,1731035408623, RpcServer on e729c2d9d3a5/172.17.0.2:45419, sessionid=0x10119b5c5050001 2024-11-08T03:10:11,128 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T03:10:11,128 DEBUG [RS:0;e729c2d9d3a5:45419 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:11,129 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e729c2d9d3a5,45419,1731035408623' 2024-11-08T03:10:11,129 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T03:10:11,130 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T03:10:11,131 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T03:10:11,131 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T03:10:11,131 DEBUG [RS:0;e729c2d9d3a5:45419 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:11,131 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e729c2d9d3a5,45419,1731035408623' 2024-11-08T03:10:11,131 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T03:10:11,132 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T03:10:11,133 DEBUG [RS:0;e729c2d9d3a5:45419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T03:10:11,133 INFO [RS:0;e729c2d9d3a5:45419 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T03:10:11,133 INFO [RS:0;e729c2d9d3a5:45419 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T03:10:11,398 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-08T03:10:11,398 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:10:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741832_1008 (size=32) 2024-11-08T03:10:11,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:11,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T03:10:11,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T03:10:11,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:11,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:11,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T03:10:11,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T03:10:11,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:11,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:11,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T03:10:11,425 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T03:10:11,425 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:11,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:11,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740 2024-11-08T03:10:11,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740 2024-11-08T03:10:11,432 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T03:10:11,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T03:10:11,438 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:10:11,439 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65269050, jitterRate=-0.027415364980697632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T03:10:11,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T03:10:11,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T03:10:11,442 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T03:10:11,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T03:10:11,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T03:10:11,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T03:10:11,443 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T03:10:11,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T03:10:11,446 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T03:10:11,446 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-08T03:10:11,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T03:10:11,458 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T03:10:11,460 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T03:10:11,612 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:12,613 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:13,614 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:14,615 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:15,616 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:16,616 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:17,618 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T03:10:18,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T03:10:18,619 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:19,620 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:20,621 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:21,622 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:22,623 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:23,625 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:24,626 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:25,628 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:26,629 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:27,630 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:28,631 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:29,633 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:30,634 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:31,635 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:32,636 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:33,637 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:34,638 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:35,639 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:36,640 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:36,936 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:10:37,114 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:10:37,641 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:38,642 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:39,642 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:40,643 WARN [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T03:10:41,139 INFO [RS:0;e729c2d9d3a5:45419 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T03:10:41,143 INFO [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e729c2d9d3a5%2C45419%2C1731035408623, suffix=, logDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623, archiveDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/oldWALs, maxLogs=32 2024-11-08T03:10:41,169 DEBUG [RS:0;e729c2d9d3a5:45419 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623/e729c2d9d3a5%2C45419%2C1731035408623.1731035441146, exclude list is [], retry=0 2024-11-08T03:10:41,177 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38891,DS-505798bb-7181-46ba-a02f-769c7cce13bc,DISK] 2024-11-08T03:10:41,180 INFO [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623/e729c2d9d3a5%2C45419%2C1731035408623.1731035441146 2024-11-08T03:10:41,181 DEBUG [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35117:35117)] 2024-11-08T03:10:41,397 DEBUG [e729c2d9d3a5:39565 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T03:10:41,405 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:41,410 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e729c2d9d3a5,45419,1731035408623, state=OPENING 2024-11-08T03:10:41,456 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T03:10:41,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:41,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:41,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T03:10:41,466 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T03:10:41,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:10:41,643 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:41,646 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T03:10:41,650 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44628, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T03:10:41,662 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-08T03:10:41,663 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T03:10:41,663 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T03:10:41,667 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e729c2d9d3a5%2C45419%2C1731035408623.meta, suffix=.meta, logDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623, archiveDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/oldWALs, maxLogs=32 2024-11-08T03:10:41,687 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623/e729c2d9d3a5%2C45419%2C1731035408623.meta.1731035441670.meta, exclude list is [], retry=0 2024-11-08T03:10:41,691 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38891,DS-505798bb-7181-46ba-a02f-769c7cce13bc,DISK] 2024-11-08T03:10:41,695 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623/e729c2d9d3a5%2C45419%2C1731035408623.meta.1731035441670.meta 2024-11-08T03:10:41,696 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35117:35117)] 2024-11-08T03:10:41,696 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:10:41,697 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T03:10:41,753 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T03:10:41,757 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T03:10:41,761 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T03:10:41,761 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:41,762 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-08T03:10:41,762 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-08T03:10:41,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T03:10:41,767 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T03:10:41,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:41,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:41,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T03:10:41,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T03:10:41,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:41,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:41,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T03:10:41,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T03:10:41,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:41,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T03:10:41,776 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740 2024-11-08T03:10:41,778 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740 2024-11-08T03:10:41,781 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T03:10:41,784 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T03:10:41,786 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63246574, jitterRate=-0.057552605867385864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T03:10:41,788 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T03:10:41,796 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731035441638 2024-11-08T03:10:41,808 DEBUG [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T03:10:41,809 INFO [RS_OPEN_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-08T03:10:41,810 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:41,812 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e729c2d9d3a5,45419,1731035408623, state=OPEN 2024-11-08T03:10:41,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T03:10:41,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T03:10:41,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T03:10:41,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T03:10:41,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T03:10:41,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=e729c2d9d3a5,45419,1731035408623 in 378 msec 2024-11-08T03:10:41,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T03:10:41,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.4030 sec 2024-11-08T03:10:41,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 30.9660 sec 2024-11-08T03:10:41,865 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731035441865, completionTime=-1 2024-11-08T03:10:41,865 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T03:10:41,865 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-08T03:10:41,899 DEBUG [hconnection-0x621ac169-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:10:41,902 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44636, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:10:41,911 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-08T03:10:41,911 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731035501911 2024-11-08T03:10:41,911 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731035561911 2024-11-08T03:10:41,912 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 46 msec 2024-11-08T03:10:41,950 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,39565,1731035407811-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:41,950 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,39565,1731035407811-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:41,950 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,39565,1731035407811-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:41,952 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e729c2d9d3a5:39565, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:41,953 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:41,959 DEBUG [master/e729c2d9d3a5:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-08T03:10:41,963 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-08T03:10:41,964 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T03:10:41,971 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-08T03:10:41,975 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:10:41,976 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:41,978 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:10:42,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741835_1011 (size=358) 2024-11-08T03:10:42,408 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 496e8ff6915da176f69774373f81da22, NAME => 'hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:10:42,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741836_1012 (size=42) 2024-11-08T03:10:42,819 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:42,820 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 496e8ff6915da176f69774373f81da22, disabling compactions & flushes 2024-11-08T03:10:42,820 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:42,820 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:42,820 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. after waiting 0 ms 2024-11-08T03:10:42,820 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:42,820 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:42,820 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 496e8ff6915da176f69774373f81da22: 2024-11-08T03:10:42,823 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:10:42,830 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731035442824"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035442824"}]},"ts":"1731035442824"} 2024-11-08T03:10:42,862 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:10:42,865 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:10:42,869 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035442865"}]},"ts":"1731035442865"} 2024-11-08T03:10:42,875 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-08T03:10:42,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=496e8ff6915da176f69774373f81da22, ASSIGN}] 2024-11-08T03:10:42,905 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=496e8ff6915da176f69774373f81da22, ASSIGN 2024-11-08T03:10:42,907 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=496e8ff6915da176f69774373f81da22, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:10:43,058 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=496e8ff6915da176f69774373f81da22, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:43,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 496e8ff6915da176f69774373f81da22, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:10:43,220 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:43,229 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:43,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 496e8ff6915da176f69774373f81da22, NAME => 'hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:10:43,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:43,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,233 INFO [StoreOpener-496e8ff6915da176f69774373f81da22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,236 INFO [StoreOpener-496e8ff6915da176f69774373f81da22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496e8ff6915da176f69774373f81da22 columnFamilyName info 2024-11-08T03:10:43,236 DEBUG [StoreOpener-496e8ff6915da176f69774373f81da22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:43,238 INFO [StoreOpener-496e8ff6915da176f69774373f81da22-1 {}] regionserver.HStore(327): Store=496e8ff6915da176f69774373f81da22/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:10:43,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,243 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 496e8ff6915da176f69774373f81da22 2024-11-08T03:10:43,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:10:43,255 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 496e8ff6915da176f69774373f81da22; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60354931, jitterRate=-0.10064144432544708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:10:43,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 496e8ff6915da176f69774373f81da22: 2024-11-08T03:10:43,260 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22., pid=6, masterSystemTime=1731035443220 2024-11-08T03:10:43,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:43,265 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:10:43,267 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=496e8ff6915da176f69774373f81da22, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:43,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T03:10:43,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 496e8ff6915da176f69774373f81da22, server=e729c2d9d3a5,45419,1731035408623 in 209 msec 2024-11-08T03:10:43,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T03:10:43,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=496e8ff6915da176f69774373f81da22, ASSIGN in 381 msec 2024-11-08T03:10:43,293 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:10:43,294 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035443293"}]},"ts":"1731035443293"} 2024-11-08T03:10:43,299 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-08T03:10:43,313 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:10:43,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3490 sec 2024-11-08T03:10:43,376 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-08T03:10:43,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:43,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-08T03:10:43,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:10:43,418 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-08T03:10:43,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T03:10:43,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 36 msec 2024-11-08T03:10:43,463 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-08T03:10:43,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T03:10:43,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 29 msec 2024-11-08T03:10:43,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-08T03:10:43,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-08T03:10:43,545 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 34.786sec 2024-11-08T03:10:43,548 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T03:10:43,550 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T03:10:43,551 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T03:10:43,552 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T03:10:43,552 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T03:10:43,554 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,39565,1731035407811-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T03:10:43,555 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,39565,1731035407811-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T03:10:43,562 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-08T03:10:43,563 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T03:10:43,563 INFO [master/e729c2d9d3a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e729c2d9d3a5,39565,1731035407811-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T03:10:43,657 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x574a453f to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61e4a9fc 2024-11-08T03:10:43,657 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-08T03:10:43,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d933e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:43,674 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T03:10:43,674 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T03:10:43,682 DEBUG [hconnection-0x35085285-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:10:43,689 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44640, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:10:43,697 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=e729c2d9d3a5,39565,1731035407811 2024-11-08T03:10:43,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=97, ProcessCount=11, AvailableMemoryMB=1624 2024-11-08T03:10:43,713 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c2cf168 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@312a75fa 2024-11-08T03:10:43,724 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0c2cf168 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3da69f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:43,728 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76ac4be3 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@144b7569 2024-11-08T03:10:43,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d25698b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:43,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T03:10:43,749 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33092, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T03:10:43,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:10:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T03:10:43,762 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:10:43,763 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:43,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-08T03:10:43,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:10:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T03:10:43,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741837_1013 (size=437) 2024-11-08T03:10:44,181 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 006d5905ba2c972f64fe852b7b6a56e7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:10:44,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741838_1014 (size=88) 2024-11-08T03:10:44,598 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:44,598 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 006d5905ba2c972f64fe852b7b6a56e7, disabling compactions & flushes 2024-11-08T03:10:44,598 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,598 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,598 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. after waiting 0 ms 2024-11-08T03:10:44,598 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,598 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,598 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 006d5905ba2c972f64fe852b7b6a56e7: 2024-11-08T03:10:44,600 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:10:44,600 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731035444600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035444600"}]},"ts":"1731035444600"} 2024-11-08T03:10:44,603 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:10:44,605 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:10:44,605 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035444605"}]},"ts":"1731035444605"} 2024-11-08T03:10:44,608 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:10:44,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=006d5905ba2c972f64fe852b7b6a56e7, ASSIGN}] 2024-11-08T03:10:44,633 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=006d5905ba2c972f64fe852b7b6a56e7, ASSIGN 2024-11-08T03:10:44,635 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=006d5905ba2c972f64fe852b7b6a56e7, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:10:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T03:10:44,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=006d5905ba2c972f64fe852b7b6a56e7, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:44,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 006d5905ba2c972f64fe852b7b6a56e7, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:10:44,950 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:44,963 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,964 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 006d5905ba2c972f64fe852b7b6a56e7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:10:44,964 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,964 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:44,965 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,965 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,967 INFO [StoreOpener-006d5905ba2c972f64fe852b7b6a56e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,970 INFO [StoreOpener-006d5905ba2c972f64fe852b7b6a56e7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 006d5905ba2c972f64fe852b7b6a56e7 columnFamilyName testFamily 2024-11-08T03:10:44,970 DEBUG [StoreOpener-006d5905ba2c972f64fe852b7b6a56e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:44,971 INFO [StoreOpener-006d5905ba2c972f64fe852b7b6a56e7-1 {}] regionserver.HStore(327): Store=006d5905ba2c972f64fe852b7b6a56e7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:10:44,973 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,973 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,977 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:10:44,980 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:10:44,981 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 006d5905ba2c972f64fe852b7b6a56e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62425633, jitterRate=-0.06978558003902435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:10:44,982 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 006d5905ba2c972f64fe852b7b6a56e7: 2024-11-08T03:10:44,984 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7., pid=11, masterSystemTime=1731035444950 2024-11-08T03:10:44,987 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,987 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:10:44,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=006d5905ba2c972f64fe852b7b6a56e7, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:44,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T03:10:44,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 006d5905ba2c972f64fe852b7b6a56e7, server=e729c2d9d3a5,45419,1731035408623 in 198 msec 2024-11-08T03:10:44,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T03:10:44,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=006d5905ba2c972f64fe852b7b6a56e7, ASSIGN in 363 msec 2024-11-08T03:10:44,999 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:10:45,000 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035444999"}]},"ts":"1731035444999"} 2024-11-08T03:10:45,002 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:10:45,012 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:10:45,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 1.2540 sec 2024-11-08T03:10:45,937 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T03:10:45,939 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33106, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T03:10:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T03:10:46,785 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-08T03:10:46,785 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:10:46,787 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:10:46,795 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T03:10:46,795 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:10:46,795 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-08T03:10:46,798 DEBUG [hconnection-0x1b039281-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:10:46,801 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44650, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:10:46,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:10:46,827 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T03:10:46,838 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:10:46,840 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0c2cf168 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:10:46,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:10:46,845 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44664, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:10:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:10:46,855 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T03:10:46,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072011 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:10:46,858 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T03:10:46,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072011 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:10:46,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479605 2024-11-08T03:10:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44664 deadline: 1731035449857, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:10:46,877 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072011 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:10:46,880 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T03:10:46,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072012 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:10:46,882 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T03:10:46,883 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:10:46,885 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:10:46,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072013 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:10:46,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479603 2024-11-08T03:10:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44664 deadline: 1731035449884, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:10:46,887 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072013 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:10:46,889 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:10:46,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072014 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:10:46,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-279807653226479602 2024-11-08T03:10:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44664 deadline: 1731035449888, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:10:46,894 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:10:46,892 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072014 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:10:46,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072015 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:10:46,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-279807653226479601 2024-11-08T03:10:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44664 deadline: 1731035449893, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:10:46,897 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072015 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:10:46,899 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:10:46,900 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76ac4be3 to 127.0.0.1:57399 2024-11-08T03:10:46,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:10:46,901 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:10:46,901 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:10:46,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:10:46,907 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c2cf168 to 127.0.0.1:57399 2024-11-08T03:10:46,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=225 (was 219) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57399@0x76ac4be3-SendThread(127.0.0.1:57399) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57399@0x0c2cf168-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: hconnection-0x35085285-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57399@0x76ac4be3 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$732/0x00007f1b9cb44bb8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57399@0x76ac4be3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57399@0x0c2cf168 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$732/0x00007f1b9cb44bb8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_590709373_22 at /127.0.0.1:55640 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57399@0x0c2cf168-SendThread(127.0.0.1:57399) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=456 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=105 (was 97) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1564 (was 1624) 2024-11-08T03:10:46,930 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=225, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=105, ProcessCount=11, AvailableMemoryMB=1564 2024-11-08T03:10:46,934 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e567203 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@488bbf84 2024-11-08T03:10:46,950 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x3e567203 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee81d62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:46,952 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b4e4ad2 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1760f201 2024-11-08T03:10:46,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1648742b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:46,965 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:10:46,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T03:10:46,969 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:10:46,969 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:46,970 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-08T03:10:46,971 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:10:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T03:10:46,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741839_1015 (size=423) 2024-11-08T03:10:47,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:10:47,346 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-08T03:10:47,347 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T03:10:47,348 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-08T03:10:47,389 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 53a46b3b2a374085c1bbdb5620bdf1e5, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:10:47,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741840_1016 (size=74) 2024-11-08T03:10:47,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:47,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 53a46b3b2a374085c1bbdb5620bdf1e5, disabling compactions & flushes 2024-11-08T03:10:47,802 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:47,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:47,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. after waiting 0 ms 2024-11-08T03:10:47,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:47,802 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:47,803 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 53a46b3b2a374085c1bbdb5620bdf1e5: 2024-11-08T03:10:47,804 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:10:47,805 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731035447804"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035447804"}]},"ts":"1731035447804"} 2024-11-08T03:10:47,808 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:10:47,809 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:10:47,809 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035447809"}]},"ts":"1731035447809"} 2024-11-08T03:10:47,812 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:10:47,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=53a46b3b2a374085c1bbdb5620bdf1e5, ASSIGN}] 2024-11-08T03:10:47,834 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=53a46b3b2a374085c1bbdb5620bdf1e5, ASSIGN 2024-11-08T03:10:47,836 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=53a46b3b2a374085c1bbdb5620bdf1e5, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:10:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T03:10:47,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=53a46b3b2a374085c1bbdb5620bdf1e5, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:47,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 53a46b3b2a374085c1bbdb5620bdf1e5, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:10:48,147 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:48,153 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:48,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => 53a46b3b2a374085c1bbdb5620bdf1e5, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:10:48,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:48,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for 53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,154 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for 53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,156 INFO [StoreOpener-53a46b3b2a374085c1bbdb5620bdf1e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,158 INFO [StoreOpener-53a46b3b2a374085c1bbdb5620bdf1e5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a46b3b2a374085c1bbdb5620bdf1e5 columnFamilyName testFamily 2024-11-08T03:10:48,158 DEBUG [StoreOpener-53a46b3b2a374085c1bbdb5620bdf1e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:48,159 INFO [StoreOpener-53a46b3b2a374085c1bbdb5620bdf1e5-1 {}] regionserver.HStore(327): Store=53a46b3b2a374085c1bbdb5620bdf1e5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:10:48,160 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,161 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,164 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for 53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:10:48,168 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:10:48,169 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened 53a46b3b2a374085c1bbdb5620bdf1e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71855350, jitterRate=0.07072815299034119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:10:48,170 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for 53a46b3b2a374085c1bbdb5620bdf1e5: 2024-11-08T03:10:48,171 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5., pid=14, masterSystemTime=1731035448147 2024-11-08T03:10:48,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:48,174 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:10:48,175 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=53a46b3b2a374085c1bbdb5620bdf1e5, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:10:48,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T03:10:48,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 53a46b3b2a374085c1bbdb5620bdf1e5, server=e729c2d9d3a5,45419,1731035408623 in 187 msec 2024-11-08T03:10:48,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-08T03:10:48,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=53a46b3b2a374085c1bbdb5620bdf1e5, ASSIGN in 350 msec 2024-11-08T03:10:48,187 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:10:48,187 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035448187"}]},"ts":"1731035448187"} 2024-11-08T03:10:48,189 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:10:48,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:10:48,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.2670 sec 2024-11-08T03:10:48,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T03:10:48,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:10:48,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T03:10:48,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T03:10:48,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T03:10:48,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-08T03:10:48,250 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T03:10:48,250 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T03:10:48,251 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T03:10:48,251 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:10:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T03:10:49,982 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-08T03:10:49,983 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:10:49,983 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:10:49,992 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T03:10:49,993 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:10:49,993 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T03:10:49,995 DEBUG [hconnection-0x512e1248-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:10:49,997 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48038, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:10:50,008 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:10:50,011 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:10:50,013 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:10:53,657 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:10:53,686 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-08T03:10:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T03:10:57,041 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:10:59,013 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:10:59,014 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T03:10:59.014Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1972ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:10:59,018 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-08T03:10:59,018 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b4e4ad2 to 127.0.0.1:57399 2024-11-08T03:10:59,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:10:59,019 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:10:59,019 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:10:59,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:10:59,019 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3e567203 to 127.0.0.1:57399 2024-11-08T03:10:59,028 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=222 (was 225), OpenFileDescriptor=448 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 105), ProcessCount=11 (was 11), AvailableMemoryMB=1533 (was 1564) 2024-11-08T03:10:59,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=1533 2024-11-08T03:10:59,038 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b8ee3d4 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51fb2da6 2024-11-08T03:10:59,062 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x2b8ee3d4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd0051b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:59,064 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c55ebef to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e7c030c 2024-11-08T03:10:59,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2230cf54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:10:59,078 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:10:59,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T03:10:59,083 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:10:59,083 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:10:59,083 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-08T03:10:59,084 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:10:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T03:10:59,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741841_1017 (size=421) 2024-11-08T03:10:59,502 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9eefc29c5ad773a11362212e16e34d43, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:10:59,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741842_1018 (size=72) 2024-11-08T03:10:59,914 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:10:59,914 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 9eefc29c5ad773a11362212e16e34d43, disabling compactions & flushes 2024-11-08T03:10:59,915 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:10:59,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:10:59,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. after waiting 0 ms 2024-11-08T03:10:59,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:10:59,915 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:10:59,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9eefc29c5ad773a11362212e16e34d43: 2024-11-08T03:10:59,919 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:10:59,920 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731035459920"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035459920"}]},"ts":"1731035459920"} 2024-11-08T03:10:59,924 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:10:59,926 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:10:59,926 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035459926"}]},"ts":"1731035459926"} 2024-11-08T03:10:59,928 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:10:59,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=9eefc29c5ad773a11362212e16e34d43, ASSIGN}] 2024-11-08T03:11:00,001 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=9eefc29c5ad773a11362212e16e34d43, ASSIGN 2024-11-08T03:11:00,003 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=9eefc29c5ad773a11362212e16e34d43, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T03:11:00,154 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=9eefc29c5ad773a11362212e16e34d43, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:00,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 9eefc29c5ad773a11362212e16e34d43, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:00,318 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:00,325 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:11:00,326 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 9eefc29c5ad773a11362212e16e34d43, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:00,326 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,326 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:00,326 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,327 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,329 INFO [StoreOpener-9eefc29c5ad773a11362212e16e34d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,332 INFO [StoreOpener-9eefc29c5ad773a11362212e16e34d43-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eefc29c5ad773a11362212e16e34d43 columnFamilyName testFamily 2024-11-08T03:11:00,332 DEBUG [StoreOpener-9eefc29c5ad773a11362212e16e34d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:00,333 INFO [StoreOpener-9eefc29c5ad773a11362212e16e34d43-1 {}] regionserver.HStore(327): Store=9eefc29c5ad773a11362212e16e34d43/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:00,334 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:11:00,342 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:00,343 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened 9eefc29c5ad773a11362212e16e34d43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72765965, jitterRate=0.08429737389087677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:00,344 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 9eefc29c5ad773a11362212e16e34d43: 2024-11-08T03:11:00,345 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43., pid=17, masterSystemTime=1731035460317 2024-11-08T03:11:00,348 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:11:00,348 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:11:00,349 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=9eefc29c5ad773a11362212e16e34d43, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:00,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-08T03:11:00,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 9eefc29c5ad773a11362212e16e34d43, server=e729c2d9d3a5,45419,1731035408623 in 188 msec 2024-11-08T03:11:00,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T03:11:00,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=9eefc29c5ad773a11362212e16e34d43, ASSIGN in 357 msec 2024-11-08T03:11:00,357 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:00,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035460357"}]},"ts":"1731035460357"} 2024-11-08T03:11:00,359 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:00,371 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:00,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 1.2920 sec 2024-11-08T03:11:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T03:11:02,095 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-08T03:11:02,095 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:02,096 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:02,104 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:02,105 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:02,105 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T03:11:02,108 DEBUG [hconnection-0x6343ed0b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:02,110 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55690, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:02,120 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:02,124 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:02,125 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:11:02,126 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x2b8ee3d4 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:11:02,126 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x2b8ee3d4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:02,128 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55702, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:11:02,133 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:11:02,133 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072027 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:02,135 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:11:02,135 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072027 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:02,136 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T03:11:03,542 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:05,852 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:05,885 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-08T03:11:06,937 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:11:08,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T03:11:08,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:11,142 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9007 2024-11-08T03:11:11,143 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c55ebef to 127.0.0.1:57399 2024-11-08T03:11:11,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:11,143 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:11,143 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072027 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:11:11,143 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:11,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:11,143 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b8ee3d4 to 127.0.0.1:57399 2024-11-08T03:11:11,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479589' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T03:11:11,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:55702 deadline: 1731035477142, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479589' does not exist 2024-11-08T03:11:11,145 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:11,143 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e729c2d9d3a5,45419,1731035408623 for closing scanner id = -279807653226479589 for 9eefc29c5ad773a11362212e16e34d43 of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T03:11:11,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=219 (was 222), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=80 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=1526 (was 1533) 2024-11-08T03:11:11,159 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=219, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=80, ProcessCount=11, AvailableMemoryMB=1526 2024-11-08T03:11:11,162 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75859c3c to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b6678da 2024-11-08T03:11:11,176 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:11:11,248 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x75859c3c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54eb985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:11,250 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58325c54 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d914229 2024-11-08T03:11:11,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142c43db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:11,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T03:11:11,268 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:11,268 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:11,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-08T03:11:11,269 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T03:11:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741843_1019 (size=432) 2024-11-08T03:11:11,637 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:11,685 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => af931b4fe418ddc9248d7e94b4f6fa28, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:11,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741844_1020 (size=83) 2024-11-08T03:11:11,991 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:11:12,099 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:12,100 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing af931b4fe418ddc9248d7e94b4f6fa28, disabling compactions & flushes 2024-11-08T03:11:12,100 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,100 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,100 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. after waiting 0 ms 2024-11-08T03:11:12,100 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,100 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,100 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for af931b4fe418ddc9248d7e94b4f6fa28: 2024-11-08T03:11:12,102 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:12,103 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731035472102"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035472102"}]},"ts":"1731035472102"} 2024-11-08T03:11:12,106 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:12,108 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:12,108 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035472108"}]},"ts":"1731035472108"} 2024-11-08T03:11:12,111 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:12,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=af931b4fe418ddc9248d7e94b4f6fa28, ASSIGN}] 2024-11-08T03:11:12,194 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=af931b4fe418ddc9248d7e94b4f6fa28, ASSIGN 2024-11-08T03:11:12,198 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=af931b4fe418ddc9248d7e94b4f6fa28, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T03:11:12,349 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=af931b4fe418ddc9248d7e94b4f6fa28, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:12,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure af931b4fe418ddc9248d7e94b4f6fa28, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:12,511 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:12,518 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,518 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => af931b4fe418ddc9248d7e94b4f6fa28, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:12,518 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,518 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:12,518 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,518 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,521 INFO [StoreOpener-af931b4fe418ddc9248d7e94b4f6fa28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,523 INFO [StoreOpener-af931b4fe418ddc9248d7e94b4f6fa28-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af931b4fe418ddc9248d7e94b4f6fa28 columnFamilyName testFamily 2024-11-08T03:11:12,523 DEBUG [StoreOpener-af931b4fe418ddc9248d7e94b4f6fa28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:12,523 INFO [StoreOpener-af931b4fe418ddc9248d7e94b4f6fa28-1 {}] regionserver.HStore(327): Store=af931b4fe418ddc9248d7e94b4f6fa28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:12,525 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,525 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,529 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:11:12,532 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:12,533 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened af931b4fe418ddc9248d7e94b4f6fa28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68828487, jitterRate=0.025624379515647888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:12,533 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for af931b4fe418ddc9248d7e94b4f6fa28: 2024-11-08T03:11:12,535 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28., pid=20, masterSystemTime=1731035472511 2024-11-08T03:11:12,537 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,537 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:11:12,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=af931b4fe418ddc9248d7e94b4f6fa28, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:12,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T03:11:12,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure af931b4fe418ddc9248d7e94b4f6fa28, server=e729c2d9d3a5,45419,1731035408623 in 184 msec 2024-11-08T03:11:12,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-08T03:11:12,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=af931b4fe418ddc9248d7e94b4f6fa28, ASSIGN in 353 msec 2024-11-08T03:11:12,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:12,546 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035472546"}]},"ts":"1731035472546"} 2024-11-08T03:11:12,548 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:12,556 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:12,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.2910 sec 2024-11-08T03:11:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T03:11:14,279 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-08T03:11:14,279 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:14,280 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:14,289 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:14,289 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:14,289 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-08T03:11:14,292 DEBUG [hconnection-0x594e8579-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:14,293 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51384, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:14,303 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:14,306 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T03:11:14,308 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T03:11:14,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072031 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:14,310 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T03:11:14,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072031 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:14,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479585 2024-11-08T03:11:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51384 deadline: 1731035475310, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:11:14,312 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:11:14,314 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T03:11:14,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072032 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:14,315 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T03:11:14,316 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:11:14,317 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:11:14,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072033 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:14,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479583 2024-11-08T03:11:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51384 deadline: 1731035475318, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:11:14,319 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:11:14,321 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:11:14,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072034 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:14,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-279807653226479582 2024-11-08T03:11:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51384 deadline: 1731035475321, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:11:14,322 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:11:14,324 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:11:14,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072035 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:14,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-279807653226479581 2024-11-08T03:11:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51384 deadline: 1731035475325, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:11:14,326 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:11:14,327 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:11:14,328 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58325c54 to 127.0.0.1:57399 2024-11-08T03:11:14,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:14,328 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:14,328 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:14,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:14,329 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75859c3c to 127.0.0.1:57399 2024-11-08T03:11:14,337 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=225 (was 219) - Thread LEAK? -, OpenFileDescriptor=454 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=80 (was 80), ProcessCount=11 (was 11), AvailableMemoryMB=1527 (was 1526) - AvailableMemoryMB LEAK? - 2024-11-08T03:11:14,344 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=80, ProcessCount=11, AvailableMemoryMB=1526 2024-11-08T03:11:14,346 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x428a041b to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d52f25d 2024-11-08T03:11:14,373 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x428a041b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37156984, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:14,374 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50cc5602 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@486b0f90 2024-11-08T03:11:14,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a53605a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:14,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T03:11:14,393 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:14,394 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:14,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-08T03:11:14,395 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T03:11:14,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741845_1021 (size=426) 2024-11-08T03:11:14,811 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8a0070350f48c6b23867fc41ac42767b, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:14,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741846_1022 (size=77) 2024-11-08T03:11:15,227 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:15,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 8a0070350f48c6b23867fc41ac42767b, disabling compactions & flushes 2024-11-08T03:11:15,228 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. after waiting 0 ms 2024-11-08T03:11:15,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,228 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,229 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8a0070350f48c6b23867fc41ac42767b: 2024-11-08T03:11:15,232 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:15,232 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731035475232"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035475232"}]},"ts":"1731035475232"} 2024-11-08T03:11:15,235 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:15,236 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:15,236 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035475236"}]},"ts":"1731035475236"} 2024-11-08T03:11:15,239 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:15,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=8a0070350f48c6b23867fc41ac42767b, ASSIGN}] 2024-11-08T03:11:15,290 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=8a0070350f48c6b23867fc41ac42767b, ASSIGN 2024-11-08T03:11:15,292 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=8a0070350f48c6b23867fc41ac42767b, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T03:11:15,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=8a0070350f48c6b23867fc41ac42767b, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:15,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 8a0070350f48c6b23867fc41ac42767b, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:15,605 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:15,616 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,616 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => 8a0070350f48c6b23867fc41ac42767b, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:15,618 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,618 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:15,618 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for 8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,618 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for 8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,622 INFO [StoreOpener-8a0070350f48c6b23867fc41ac42767b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,625 INFO [StoreOpener-8a0070350f48c6b23867fc41ac42767b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a0070350f48c6b23867fc41ac42767b columnFamilyName testFamily 2024-11-08T03:11:15,625 DEBUG [StoreOpener-8a0070350f48c6b23867fc41ac42767b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:15,626 INFO [StoreOpener-8a0070350f48c6b23867fc41ac42767b-1 {}] regionserver.HStore(327): Store=8a0070350f48c6b23867fc41ac42767b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:15,628 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,629 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,632 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for 8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:11:15,636 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:15,637 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened 8a0070350f48c6b23867fc41ac42767b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68204675, jitterRate=0.016328856348991394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:15,638 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for 8a0070350f48c6b23867fc41ac42767b: 2024-11-08T03:11:15,640 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b., pid=23, masterSystemTime=1731035475605 2024-11-08T03:11:15,642 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,642 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:11:15,643 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=8a0070350f48c6b23867fc41ac42767b, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:15,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-08T03:11:15,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 8a0070350f48c6b23867fc41ac42767b, server=e729c2d9d3a5,45419,1731035408623 in 195 msec 2024-11-08T03:11:15,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T03:11:15,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=8a0070350f48c6b23867fc41ac42767b, ASSIGN in 360 msec 2024-11-08T03:11:15,652 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:15,652 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035475652"}]},"ts":"1731035475652"} 2024-11-08T03:11:15,654 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:15,664 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:15,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.2740 sec 2024-11-08T03:11:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T03:11:17,404 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-08T03:11:17,404 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:17,404 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:17,411 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:17,412 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:17,412 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T03:11:17,414 DEBUG [hconnection-0x2350e8e7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:17,416 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51392, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:17,429 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:17,432 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:17,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:11:17,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T03:11:18,043 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:18,080 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-08T03:11:18,081 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-08T03:11:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T03:11:18,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T03:11:18,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T03:11:19,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:11:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51392 deadline: 1731035480435, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:11:19,457 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:11:19,458 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072040 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:11:19,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2026 2024-11-08T03:11:19,459 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50cc5602 to 127.0.0.1:57399 2024-11-08T03:11:19,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:19,460 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:19,460 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:19,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:19,460 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x428a041b to 127.0.0.1:57399 2024-11-08T03:11:19,469 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=74 (was 80), ProcessCount=11 (was 11), AvailableMemoryMB=1490 (was 1526) 2024-11-08T03:11:19,477 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=74, ProcessCount=11, AvailableMemoryMB=1489 2024-11-08T03:11:19,479 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x44175fc0 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46f576b6 2024-11-08T03:11:19,489 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x44175fc0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709ae036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:19,491 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b1012c8 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c8e56da 2024-11-08T03:11:19,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d9be8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:19,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T03:11:19,511 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:19,511 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:19,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-08T03:11:19,512 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T03:11:19,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741847_1023 (size=427) 2024-11-08T03:11:19,926 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 64a03bd514341f2ca7c009db1aa26e9d, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:19,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741848_1024 (size=78) 2024-11-08T03:11:20,341 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:20,342 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 64a03bd514341f2ca7c009db1aa26e9d, disabling compactions & flushes 2024-11-08T03:11:20,342 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,342 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,342 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. after waiting 0 ms 2024-11-08T03:11:20,342 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,342 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,342 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 64a03bd514341f2ca7c009db1aa26e9d: 2024-11-08T03:11:20,345 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:20,345 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731035480345"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035480345"}]},"ts":"1731035480345"} 2024-11-08T03:11:20,348 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:20,349 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:20,350 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035480350"}]},"ts":"1731035480350"} 2024-11-08T03:11:20,352 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:20,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=64a03bd514341f2ca7c009db1aa26e9d, ASSIGN}] 2024-11-08T03:11:20,498 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=64a03bd514341f2ca7c009db1aa26e9d, ASSIGN 2024-11-08T03:11:20,499 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=64a03bd514341f2ca7c009db1aa26e9d, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T03:11:20,650 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=64a03bd514341f2ca7c009db1aa26e9d, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:20,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 64a03bd514341f2ca7c009db1aa26e9d, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:20,810 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:20,821 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,822 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => 64a03bd514341f2ca7c009db1aa26e9d, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:20,822 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,822 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:20,823 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for 64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,823 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for 64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,825 INFO [StoreOpener-64a03bd514341f2ca7c009db1aa26e9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,828 INFO [StoreOpener-64a03bd514341f2ca7c009db1aa26e9d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64a03bd514341f2ca7c009db1aa26e9d columnFamilyName testFamily 2024-11-08T03:11:20,828 DEBUG [StoreOpener-64a03bd514341f2ca7c009db1aa26e9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:20,829 INFO [StoreOpener-64a03bd514341f2ca7c009db1aa26e9d-1 {}] regionserver.HStore(327): Store=64a03bd514341f2ca7c009db1aa26e9d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:20,831 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,832 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,836 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for 64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:11:20,839 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:20,840 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened 64a03bd514341f2ca7c009db1aa26e9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63666598, jitterRate=-0.051293760538101196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:20,841 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for 64a03bd514341f2ca7c009db1aa26e9d: 2024-11-08T03:11:20,842 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d., pid=26, masterSystemTime=1731035480810 2024-11-08T03:11:20,845 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,845 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:11:20,846 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=64a03bd514341f2ca7c009db1aa26e9d, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:20,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T03:11:20,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 64a03bd514341f2ca7c009db1aa26e9d, server=e729c2d9d3a5,45419,1731035408623 in 193 msec 2024-11-08T03:11:20,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T03:11:20,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=64a03bd514341f2ca7c009db1aa26e9d, ASSIGN in 354 msec 2024-11-08T03:11:20,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:20,854 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035480854"}]},"ts":"1731035480854"} 2024-11-08T03:11:20,856 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:20,899 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:20,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 1.3920 sec 2024-11-08T03:11:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T03:11:22,522 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-08T03:11:22,522 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:22,522 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:22,528 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:22,528 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:22,528 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T03:11:22,530 DEBUG [hconnection-0x32e6afd9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:22,532 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45700, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:22,542 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:22,545 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:11:22,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:11:23,588 INFO [master/e729c2d9d3a5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T03:11:23,588 INFO [master/e729c2d9d3a5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T03:11:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T03:11:24,083 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-08T03:11:24,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:11:25,553 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:11:25,553 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T03:11:25.553Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:11:25,555 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3010 2024-11-08T03:11:25,556 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b1012c8 to 127.0.0.1:57399 2024-11-08T03:11:25,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:25,556 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:25,556 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:25,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:25,557 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x44175fc0 to 127.0.0.1:57399 2024-11-08T03:11:25,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225 (was 227), OpenFileDescriptor=455 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=107 (was 74) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1500 (was 1489) - AvailableMemoryMB LEAK? - 2024-11-08T03:11:25,581 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=107, ProcessCount=11, AvailableMemoryMB=1499 2024-11-08T03:11:25,584 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50ca2cd2 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29e3c0fe 2024-11-08T03:11:25,598 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x50ca2cd2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4cbc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:25,601 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4dd486c6 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a698487 2024-11-08T03:11:25,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b449ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:25,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T03:11:25,619 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:25,619 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:25,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-08T03:11:25,620 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T03:11:25,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741849_1025 (size=425) 2024-11-08T03:11:26,032 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e4dc35a373b38f6d59efad851dae653f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:26,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741850_1026 (size=76) 2024-11-08T03:11:26,067 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:26,326 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:26,445 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:26,445 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing e4dc35a373b38f6d59efad851dae653f, disabling compactions & flushes 2024-11-08T03:11:26,445 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,445 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,445 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. after waiting 0 ms 2024-11-08T03:11:26,445 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,445 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,445 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for e4dc35a373b38f6d59efad851dae653f: 2024-11-08T03:11:26,447 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:26,447 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731035486447"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035486447"}]},"ts":"1731035486447"} 2024-11-08T03:11:26,449 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:26,450 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:26,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035486450"}]},"ts":"1731035486450"} 2024-11-08T03:11:26,453 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:26,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=e4dc35a373b38f6d59efad851dae653f, ASSIGN}] 2024-11-08T03:11:26,473 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=e4dc35a373b38f6d59efad851dae653f, ASSIGN 2024-11-08T03:11:26,474 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=e4dc35a373b38f6d59efad851dae653f, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T03:11:26,624 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=e4dc35a373b38f6d59efad851dae653f, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:26,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure e4dc35a373b38f6d59efad851dae653f, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:26,780 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:26,789 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,790 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => e4dc35a373b38f6d59efad851dae653f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:26,790 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,790 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:26,791 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,791 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,793 INFO [StoreOpener-e4dc35a373b38f6d59efad851dae653f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,796 INFO [StoreOpener-e4dc35a373b38f6d59efad851dae653f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4dc35a373b38f6d59efad851dae653f columnFamilyName testFamily 2024-11-08T03:11:26,796 DEBUG [StoreOpener-e4dc35a373b38f6d59efad851dae653f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:26,797 INFO [StoreOpener-e4dc35a373b38f6d59efad851dae653f-1 {}] regionserver.HStore(327): Store=e4dc35a373b38f6d59efad851dae653f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:26,798 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,799 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,802 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:11:26,806 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:26,807 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened e4dc35a373b38f6d59efad851dae653f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75394559, jitterRate=0.12346647679805756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:26,807 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for e4dc35a373b38f6d59efad851dae653f: 2024-11-08T03:11:26,808 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f., pid=29, masterSystemTime=1731035486780 2024-11-08T03:11:26,810 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,810 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:11:26,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=e4dc35a373b38f6d59efad851dae653f, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:26,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T03:11:26,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure e4dc35a373b38f6d59efad851dae653f, server=e729c2d9d3a5,45419,1731035408623 in 186 msec 2024-11-08T03:11:26,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-08T03:11:26,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=e4dc35a373b38f6d59efad851dae653f, ASSIGN in 344 msec 2024-11-08T03:11:26,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:26,820 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035486820"}]},"ts":"1731035486820"} 2024-11-08T03:11:26,822 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:26,831 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:26,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.2140 sec 2024-11-08T03:11:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T03:11:28,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:28,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T03:11:28,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T03:11:28,629 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-08T03:11:28,629 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:28,630 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:28,636 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:28,636 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:28,636 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T03:11:28,638 DEBUG [hconnection-0x4ca1d81b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:28,640 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39436, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:28,655 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:28,658 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:28,659 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:11:28,660 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x50ca2cd2 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:11:28,660 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x50ca2cd2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:28,662 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39442, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:11:28,665 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:11:28,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072052 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:28,667 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:11:28,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072052 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:28,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T03:11:30,089 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-08T03:11:31,673 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3006 2024-11-08T03:11:31,674 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4dd486c6 to 127.0.0.1:57399 2024-11-08T03:11:31,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:31,674 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:31,674 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072052 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:11:31,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:31,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:31,676 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50ca2cd2 to 127.0.0.1:57399 2024-11-08T03:11:31,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479564' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T03:11:31,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:39442 deadline: 1731035492673, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479564' does not exist 2024-11-08T03:11:31,680 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:31,676 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e729c2d9d3a5,45419,1731035408623 for closing scanner id = -279807653226479564 for e4dc35a373b38f6d59efad851dae653f of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T03:11:31,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=452 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=107 (was 107), ProcessCount=11 (was 11), AvailableMemoryMB=1492 (was 1499) 2024-11-08T03:11:31,695 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=107, ProcessCount=11, AvailableMemoryMB=1492 2024-11-08T03:11:31,697 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x36cd47d5 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a1dbdc4 2024-11-08T03:11:31,722 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x36cd47d5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3356d76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:31,725 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d75d0eb to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4feba97d 2024-11-08T03:11:31,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1344acba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:31,741 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:31,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-08T03:11:31,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:31,745 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:31,745 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-08T03:11:31,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T03:11:31,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741851_1027 (size=417) 2024-11-08T03:11:32,164 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3d77e8660e3bb21e5a6ae8c9b8811473, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:32,168 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:32,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741852_1028 (size=68) 2024-11-08T03:11:32,293 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:32,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:32,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 3d77e8660e3bb21e5a6ae8c9b8811473, disabling compactions & flushes 2024-11-08T03:11:32,578 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,578 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,578 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. after waiting 0 ms 2024-11-08T03:11:32,578 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,578 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,579 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3d77e8660e3bb21e5a6ae8c9b8811473: 2024-11-08T03:11:32,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:32,582 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731035492582"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035492582"}]},"ts":"1731035492582"} 2024-11-08T03:11:32,587 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:32,589 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:32,589 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035492589"}]},"ts":"1731035492589"} 2024-11-08T03:11:32,591 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:32,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3d77e8660e3bb21e5a6ae8c9b8811473, ASSIGN}] 2024-11-08T03:11:32,649 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3d77e8660e3bb21e5a6ae8c9b8811473, ASSIGN 2024-11-08T03:11:32,652 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3d77e8660e3bb21e5a6ae8c9b8811473, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T03:11:32,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=3d77e8660e3bb21e5a6ae8c9b8811473, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:32,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 3d77e8660e3bb21e5a6ae8c9b8811473, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:32,961 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:32,971 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,971 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 3d77e8660e3bb21e5a6ae8c9b8811473, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:32,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:32,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,975 INFO [StoreOpener-3d77e8660e3bb21e5a6ae8c9b8811473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,978 INFO [StoreOpener-3d77e8660e3bb21e5a6ae8c9b8811473-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d77e8660e3bb21e5a6ae8c9b8811473 columnFamilyName testFamily 2024-11-08T03:11:32,978 DEBUG [StoreOpener-3d77e8660e3bb21e5a6ae8c9b8811473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:32,979 INFO [StoreOpener-3d77e8660e3bb21e5a6ae8c9b8811473-1 {}] regionserver.HStore(327): Store=3d77e8660e3bb21e5a6ae8c9b8811473/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:32,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,983 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:11:32,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:32,986 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 3d77e8660e3bb21e5a6ae8c9b8811473; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64346444, jitterRate=-0.04116326570510864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:32,987 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 3d77e8660e3bb21e5a6ae8c9b8811473: 2024-11-08T03:11:32,988 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473., pid=32, masterSystemTime=1731035492961 2024-11-08T03:11:32,990 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,990 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:11:32,990 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=3d77e8660e3bb21e5a6ae8c9b8811473, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:32,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T03:11:32,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 3d77e8660e3bb21e5a6ae8c9b8811473, server=e729c2d9d3a5,45419,1731035408623 in 185 msec 2024-11-08T03:11:32,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T03:11:32,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3d77e8660e3bb21e5a6ae8c9b8811473, ASSIGN in 349 msec 2024-11-08T03:11:32,998 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:32,998 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035492998"}]},"ts":"1731035492998"} 2024-11-08T03:11:33,000 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:33,005 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:33,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 1.2630 sec 2024-11-08T03:11:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T03:11:34,754 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-08T03:11:34,755 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:34,755 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:34,763 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:34,764 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:34,764 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-08T03:11:34,766 DEBUG [hconnection-0x23f418cc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:34,768 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39446, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:34,778 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:34,781 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:34,784 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072056 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:11:34,784 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T03:11:36,093 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-08T03:11:36,937 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:11:38,476 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:38,510 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T03:11:38,510 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:41,286 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6504 2024-11-08T03:11:41,286 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d75d0eb to 127.0.0.1:57399 2024-11-08T03:11:41,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:41,286 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:41,286 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:41,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:41,287 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x36cd47d5 to 127.0.0.1:57399 2024-11-08T03:11:41,299 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 227) - Thread LEAK? -, OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=90 (was 107), ProcessCount=11 (was 11), AvailableMemoryMB=1478 (was 1492) 2024-11-08T03:11:41,310 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=90, ProcessCount=11, AvailableMemoryMB=1478 2024-11-08T03:11:41,313 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32cfbf1f to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@299eb4a0 2024-11-08T03:11:41,321 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x32cfbf1f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a476cb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:41,323 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73e13866 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a2301ef 2024-11-08T03:11:41,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655b74f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:41,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-08T03:11:41,334 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:41,334 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:41,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-08T03:11:41,336 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T03:11:41,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741853_1029 (size=421) 2024-11-08T03:11:41,752 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5d61dadc50789866d6f56149ebb69543, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:41,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741854_1030 (size=72) 2024-11-08T03:11:41,959 DEBUG [master/e729c2d9d3a5:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T03:11:41,969 DEBUG [master/e729c2d9d3a5:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 496e8ff6915da176f69774373f81da22 changed from -1.0 to 0.0, refreshing cache 2024-11-08T03:11:42,167 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:42,168 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 5d61dadc50789866d6f56149ebb69543, disabling compactions & flushes 2024-11-08T03:11:42,168 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,168 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,168 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. after waiting 0 ms 2024-11-08T03:11:42,168 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,168 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,168 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5d61dadc50789866d6f56149ebb69543: 2024-11-08T03:11:42,171 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:42,172 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731035502172"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035502172"}]},"ts":"1731035502172"} 2024-11-08T03:11:42,175 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:42,176 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:42,176 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035502176"}]},"ts":"1731035502176"} 2024-11-08T03:11:42,178 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:42,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=5d61dadc50789866d6f56149ebb69543, ASSIGN}] 2024-11-08T03:11:42,230 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=5d61dadc50789866d6f56149ebb69543, ASSIGN 2024-11-08T03:11:42,231 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=5d61dadc50789866d6f56149ebb69543, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T03:11:42,382 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=5d61dadc50789866d6f56149ebb69543, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:42,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 5d61dadc50789866d6f56149ebb69543, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:42,535 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:11:42,544 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:42,557 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,558 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => 5d61dadc50789866d6f56149ebb69543, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:42,559 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,559 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:42,560 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for 5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,560 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for 5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,563 INFO [StoreOpener-5d61dadc50789866d6f56149ebb69543-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,565 INFO [StoreOpener-5d61dadc50789866d6f56149ebb69543-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d61dadc50789866d6f56149ebb69543 columnFamilyName testFamily 2024-11-08T03:11:42,565 DEBUG [StoreOpener-5d61dadc50789866d6f56149ebb69543-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:42,566 INFO [StoreOpener-5d61dadc50789866d6f56149ebb69543-1 {}] regionserver.HStore(327): Store=5d61dadc50789866d6f56149ebb69543/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:42,567 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,568 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,571 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for 5d61dadc50789866d6f56149ebb69543 2024-11-08T03:11:42,574 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:42,574 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened 5d61dadc50789866d6f56149ebb69543; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73200968, jitterRate=0.09077942371368408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:42,575 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for 5d61dadc50789866d6f56149ebb69543: 2024-11-08T03:11:42,576 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543., pid=35, masterSystemTime=1731035502544 2024-11-08T03:11:42,578 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,578 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:11:42,579 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=5d61dadc50789866d6f56149ebb69543, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:42,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-08T03:11:42,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 5d61dadc50789866d6f56149ebb69543, server=e729c2d9d3a5,45419,1731035408623 in 193 msec 2024-11-08T03:11:42,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T03:11:42,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=5d61dadc50789866d6f56149ebb69543, ASSIGN in 354 msec 2024-11-08T03:11:42,586 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:42,586 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035502586"}]},"ts":"1731035502586"} 2024-11-08T03:11:42,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:42,596 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:42,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 1.2640 sec 2024-11-08T03:11:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T03:11:44,344 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-08T03:11:44,345 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:44,345 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:44,353 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:44,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:44,353 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-08T03:11:44,355 DEBUG [hconnection-0x4fb899cf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:44,357 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42940, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:44,365 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:44,368 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:44,371 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072060 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:11:44,371 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T03:11:44,402 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:11:45,873 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1504 2024-11-08T03:11:45,873 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x73e13866 to 127.0.0.1:57399 2024-11-08T03:11:45,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:45,874 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:45,874 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:45,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:45,875 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32cfbf1f to 127.0.0.1:57399 2024-11-08T03:11:45,890 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=230 (was 228) - Thread LEAK? -, OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=83 (was 90), ProcessCount=11 (was 11), AvailableMemoryMB=1477 (was 1478) 2024-11-08T03:11:45,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=83, ProcessCount=11, AvailableMemoryMB=1477 2024-11-08T03:11:45,904 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x72f683b9 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f826340 2024-11-08T03:11:45,948 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x72f683b9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edf063c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:45,952 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b598a81 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a8fcc67 2024-11-08T03:11:45,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c2267c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:45,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-08T03:11:45,986 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:45,987 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:45,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-08T03:11:45,987 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T03:11:45,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741855_1031 (size=420) 2024-11-08T03:11:46,400 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c74b6a41d3522743e243c16780bdd0c7, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741856_1032 (size=71) 2024-11-08T03:11:46,814 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:46,814 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing c74b6a41d3522743e243c16780bdd0c7, disabling compactions & flushes 2024-11-08T03:11:46,815 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:46,815 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:46,815 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. after waiting 0 ms 2024-11-08T03:11:46,815 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:46,815 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:46,815 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for c74b6a41d3522743e243c16780bdd0c7: 2024-11-08T03:11:46,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:46,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731035506818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035506818"}]},"ts":"1731035506818"} 2024-11-08T03:11:46,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:46,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:46,823 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035506822"}]},"ts":"1731035506822"} 2024-11-08T03:11:46,825 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:46,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=c74b6a41d3522743e243c16780bdd0c7, ASSIGN}] 2024-11-08T03:11:46,872 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=c74b6a41d3522743e243c16780bdd0c7, ASSIGN 2024-11-08T03:11:46,873 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=c74b6a41d3522743e243c16780bdd0c7, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T03:11:47,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=c74b6a41d3522743e243c16780bdd0c7, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:47,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure c74b6a41d3522743e243c16780bdd0c7, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:47,185 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:47,196 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:47,196 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => c74b6a41d3522743e243c16780bdd0c7, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:47,197 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,197 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:47,198 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,198 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,200 INFO [StoreOpener-c74b6a41d3522743e243c16780bdd0c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,202 INFO [StoreOpener-c74b6a41d3522743e243c16780bdd0c7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c74b6a41d3522743e243c16780bdd0c7 columnFamilyName testFamily 2024-11-08T03:11:47,202 DEBUG [StoreOpener-c74b6a41d3522743e243c16780bdd0c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:47,203 INFO [StoreOpener-c74b6a41d3522743e243c16780bdd0c7-1 {}] regionserver.HStore(327): Store=c74b6a41d3522743e243c16780bdd0c7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:47,204 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,205 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,208 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:11:47,211 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:47,212 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened c74b6a41d3522743e243c16780bdd0c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63615081, jitterRate=-0.052061423659324646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:47,214 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for c74b6a41d3522743e243c16780bdd0c7: 2024-11-08T03:11:47,215 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7., pid=38, masterSystemTime=1731035507185 2024-11-08T03:11:47,217 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:47,217 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:11:47,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=c74b6a41d3522743e243c16780bdd0c7, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:47,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-08T03:11:47,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure c74b6a41d3522743e243c16780bdd0c7, server=e729c2d9d3a5,45419,1731035408623 in 189 msec 2024-11-08T03:11:47,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T03:11:47,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=c74b6a41d3522743e243c16780bdd0c7, ASSIGN in 352 msec 2024-11-08T03:11:47,225 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:47,225 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035507225"}]},"ts":"1731035507225"} 2024-11-08T03:11:47,226 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:47,238 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:47,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 1.2550 sec 2024-11-08T03:11:48,063 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:48,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-08T03:11:48,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-08T03:11:48,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T03:11:48,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:48,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T03:11:48,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T03:11:48,998 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-08T03:11:48,998 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:48,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:49,009 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:49,010 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:49,010 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-08T03:11:49,012 DEBUG [hconnection-0x6cb755ff-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:49,014 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56348, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:49,022 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:49,026 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f2458c9 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e1b2828 2024-11-08T03:11:49,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccf54f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:49,063 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:49,065 DEBUG [hconnection-0xa427c47-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:49,067 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56362, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:49,070 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:11:49,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072065 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:49,072 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:11:49,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072065 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:11:49,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T03:11:50,083 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T03:11:50.083Z, RpcRetryingCaller{globalStartTime=2024-11-08T03:11:49.072Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T03:11:50.083Z, RpcRetryingCaller{globalStartTime=2024-11-08T03:11:49.072Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:11:50,084 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1012 2024-11-08T03:11:50,085 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f2458c9 to 127.0.0.1:57399 2024-11-08T03:11:50,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:50,086 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b598a81 to 127.0.0.1:57399 2024-11-08T03:11:50,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:50,087 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:11:50,087 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:11:50,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:11:50,088 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x72f683b9 to 127.0.0.1:57399 2024-11-08T03:11:50,098 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 230) - Thread LEAK? -, OpenFileDescriptor=456 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=83 (was 83), ProcessCount=11 (was 11), AvailableMemoryMB=1470 (was 1477) 2024-11-08T03:11:50,106 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=83, ProcessCount=11, AvailableMemoryMB=1469 2024-11-08T03:11:50,108 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53f60566 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72dcb229 2024-11-08T03:11:50,130 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x53f60566 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9669d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:50,132 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75f96e59 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69e15132 2024-11-08T03:11:50,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@adfb45d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:11:50,151 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:11:50,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T03:11:50,154 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:11:50,154 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:50,154 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-08T03:11:50,156 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:11:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T03:11:50,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741857_1033 (size=422) 2024-11-08T03:11:50,573 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 981c15823b40f09b5aafa63cd023e8c0, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:11:50,574 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:11:50,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741858_1034 (size=73) 2024-11-08T03:11:50,988 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:50,988 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 981c15823b40f09b5aafa63cd023e8c0, disabling compactions & flushes 2024-11-08T03:11:50,988 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:50,988 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:50,989 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. after waiting 0 ms 2024-11-08T03:11:50,989 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:50,989 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:50,989 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 981c15823b40f09b5aafa63cd023e8c0: 2024-11-08T03:11:51,015 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:11:51,016 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731035511015"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035511015"}]},"ts":"1731035511015"} 2024-11-08T03:11:51,020 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:11:51,021 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:11:51,022 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035511021"}]},"ts":"1731035511021"} 2024-11-08T03:11:51,023 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:11:51,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=981c15823b40f09b5aafa63cd023e8c0, ASSIGN}] 2024-11-08T03:11:51,055 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=981c15823b40f09b5aafa63cd023e8c0, ASSIGN 2024-11-08T03:11:51,056 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=981c15823b40f09b5aafa63cd023e8c0, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:11:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T03:11:51,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=981c15823b40f09b5aafa63cd023e8c0, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:51,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 981c15823b40f09b5aafa63cd023e8c0, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:11:51,363 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:51,369 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:51,369 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 981c15823b40f09b5aafa63cd023e8c0, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:11:51,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:11:51,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,373 INFO [StoreOpener-981c15823b40f09b5aafa63cd023e8c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,375 INFO [StoreOpener-981c15823b40f09b5aafa63cd023e8c0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 981c15823b40f09b5aafa63cd023e8c0 columnFamilyName testFamily 2024-11-08T03:11:51,375 DEBUG [StoreOpener-981c15823b40f09b5aafa63cd023e8c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:11:51,376 INFO [StoreOpener-981c15823b40f09b5aafa63cd023e8c0-1 {}] regionserver.HStore(327): Store=981c15823b40f09b5aafa63cd023e8c0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:11:51,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:11:51,385 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:11:51,386 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 981c15823b40f09b5aafa63cd023e8c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73532792, jitterRate=0.09572398662567139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:11:51,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 981c15823b40f09b5aafa63cd023e8c0: 2024-11-08T03:11:51,388 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0., pid=41, masterSystemTime=1731035511363 2024-11-08T03:11:51,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:51,390 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:11:51,390 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=981c15823b40f09b5aafa63cd023e8c0, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:11:51,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T03:11:51,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 981c15823b40f09b5aafa63cd023e8c0, server=e729c2d9d3a5,45419,1731035408623 in 182 msec 2024-11-08T03:11:51,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-08T03:11:51,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=981c15823b40f09b5aafa63cd023e8c0, ASSIGN in 341 msec 2024-11-08T03:11:51,397 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:11:51,397 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035511397"}]},"ts":"1731035511397"} 2024-11-08T03:11:51,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:11:51,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:11:51,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.2530 sec 2024-11-08T03:11:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T03:11:53,165 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-08T03:11:53,166 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:11:53,167 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:53,175 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T03:11:53,175 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:11:53,176 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T03:11:53,178 DEBUG [hconnection-0x37b1a809-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:11:53,179 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56376, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:11:53,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:11:53,191 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:11:53,193 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:11:53,193 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T03:11:54,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-08T03:11:56,524 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479596 expired clientIPAndPort=172.17.0.2:48038, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:11:56,874 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:11:58,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T03:11:58,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:11:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T03:12:00,219 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:12:00,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:56376 deadline: 1731035522193, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:12:00,220 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:12:00,221 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072069 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:12:00,222 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7030 2024-11-08T03:12:00,222 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75f96e59 to 127.0.0.1:57399 2024-11-08T03:12:00,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:00,222 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:00,223 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:00,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:00,223 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x53f60566 to 127.0.0.1:57399 2024-11-08T03:12:00,234 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=231 (was 236), OpenFileDescriptor=450 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=87 (was 83) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1460 (was 1469) 2024-11-08T03:12:00,242 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=87, ProcessCount=11, AvailableMemoryMB=1460 2024-11-08T03:12:00,243 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d771c6e to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1345e320 2024-11-08T03:12:00,280 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x1d771c6e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd8fb39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:00,284 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48186989 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f928a53 2024-11-08T03:12:00,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2379a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:00,300 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:00,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T03:12:00,303 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:00,303 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:00,303 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-08T03:12:00,304 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T03:12:00,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741859_1035 (size=428) 2024-11-08T03:12:00,720 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8874f3ae66c0b430c92fcb070abb9ee1, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741860_1036 (size=79) 2024-11-08T03:12:01,134 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:01,134 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 8874f3ae66c0b430c92fcb070abb9ee1, disabling compactions & flushes 2024-11-08T03:12:01,134 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,134 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,134 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. after waiting 0 ms 2024-11-08T03:12:01,134 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,134 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,135 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8874f3ae66c0b430c92fcb070abb9ee1: 2024-11-08T03:12:01,136 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:01,136 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731035521136"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035521136"}]},"ts":"1731035521136"} 2024-11-08T03:12:01,139 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:01,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:01,140 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035521140"}]},"ts":"1731035521140"} 2024-11-08T03:12:01,142 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:12:01,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8874f3ae66c0b430c92fcb070abb9ee1, ASSIGN}] 2024-11-08T03:12:01,189 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8874f3ae66c0b430c92fcb070abb9ee1, ASSIGN 2024-11-08T03:12:01,191 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8874f3ae66c0b430c92fcb070abb9ee1, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T03:12:01,342 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=8874f3ae66c0b430c92fcb070abb9ee1, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:01,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 8874f3ae66c0b430c92fcb070abb9ee1, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:01,498 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:01,503 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 8874f3ae66c0b430c92fcb070abb9ee1, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:01,504 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,504 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:01,504 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,504 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,506 INFO [StoreOpener-8874f3ae66c0b430c92fcb070abb9ee1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,507 INFO [StoreOpener-8874f3ae66c0b430c92fcb070abb9ee1-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8874f3ae66c0b430c92fcb070abb9ee1 columnFamilyName testFamily 2024-11-08T03:12:01,507 DEBUG [StoreOpener-8874f3ae66c0b430c92fcb070abb9ee1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:01,508 INFO [StoreOpener-8874f3ae66c0b430c92fcb070abb9ee1-1 {}] regionserver.HStore(327): Store=8874f3ae66c0b430c92fcb070abb9ee1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:01,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,512 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:12:01,514 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:01,515 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 8874f3ae66c0b430c92fcb070abb9ee1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69568801, jitterRate=0.03665591776371002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:01,516 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 8874f3ae66c0b430c92fcb070abb9ee1: 2024-11-08T03:12:01,517 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., pid=44, masterSystemTime=1731035521498 2024-11-08T03:12:01,518 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,519 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:12:01,519 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=8874f3ae66c0b430c92fcb070abb9ee1, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:01,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-08T03:12:01,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 8874f3ae66c0b430c92fcb070abb9ee1, server=e729c2d9d3a5,45419,1731035408623 in 175 msec 2024-11-08T03:12:01,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T03:12:01,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8874f3ae66c0b430c92fcb070abb9ee1, ASSIGN in 336 msec 2024-11-08T03:12:01,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:01,526 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035521526"}]},"ts":"1731035521526"} 2024-11-08T03:12:01,527 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:12:01,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:01,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.2370 sec 2024-11-08T03:12:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T03:12:03,312 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-08T03:12:03,312 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:12:03,312 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:03,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T03:12:03,317 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:03,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T03:12:03,319 DEBUG [hconnection-0x175cb2b1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:03,321 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55006, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:03,329 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:03,332 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:12:03,332 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:12:03,333 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x1d771c6e {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:12:03,334 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x1d771c6e {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:03,335 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55022, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:12:03,337 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:12:03,634 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479594 expired clientIPAndPort=172.17.0.2:48038, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:12:06,105 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T03:12:06,938 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:12:07,008 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:12:08,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T03:12:08,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:09,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2024-11-08T03:12:09,346 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2024-11-08T03:12:09,346 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:12:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T03:12:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:12:10,370 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:12:11,646 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479589 expired clientIPAndPort=172.17.0.2:55702, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:12:12,342 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1974ms, rpcTimeout=1964ms 2024-11-08T03:12:12,343 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1974ms, rpcTimeout=1964ms 2024-11-08T03:12:12,343 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:12:12,344 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T03:12:09.347Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2024-11-08T03:12:12.343Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1974ms, rpcTimeout=1964ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1974ms, rpcTimeout=1964ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1974ms, rpcTimeout=1964ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T03:12:12,348 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9015 2024-11-08T03:12:12,349 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x48186989 to 127.0.0.1:57399 2024-11-08T03:12:12,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:12,349 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:12,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:12,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:12,350 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d771c6e to 127.0.0.1:57399 2024-11-08T03:12:12,362 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229 (was 231), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=73 (was 87), ProcessCount=11 (was 11), AvailableMemoryMB=1445 (was 1460) 2024-11-08T03:12:12,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=229, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=73, ProcessCount=11, AvailableMemoryMB=1445 2024-11-08T03:12:12,371 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x00a0e159 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4be60061 2024-11-08T03:12:12,430 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x00a0e159 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158cb7fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:12,436 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1a2f0601 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b590599 2024-11-08T03:12:12,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf49d82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:12,448 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:12,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-08T03:12:12,450 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:12,450 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:12,450 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-08T03:12:12,451 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T03:12:12,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741861_1037 (size=416) 2024-11-08T03:12:12,577 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:12:12,866 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4b725c0119f334af117396c8418b7162, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741862_1038 (size=67) 2024-11-08T03:12:13,305 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:13,305 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 4b725c0119f334af117396c8418b7162, disabling compactions & flushes 2024-11-08T03:12:13,305 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,306 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,306 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. after waiting 0 ms 2024-11-08T03:12:13,306 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,306 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,306 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4b725c0119f334af117396c8418b7162: 2024-11-08T03:12:13,308 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:13,309 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731035533308"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035533308"}]},"ts":"1731035533308"} 2024-11-08T03:12:13,311 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:13,312 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:13,313 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035533313"}]},"ts":"1731035533313"} 2024-11-08T03:12:13,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:12:13,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=4b725c0119f334af117396c8418b7162, ASSIGN}] 2024-11-08T03:12:13,365 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=4b725c0119f334af117396c8418b7162, ASSIGN 2024-11-08T03:12:13,367 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=4b725c0119f334af117396c8418b7162, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T03:12:13,518 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=4b725c0119f334af117396c8418b7162, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:13,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 4b725c0119f334af117396c8418b7162, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:13,674 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:13,679 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 4b725c0119f334af117396c8418b7162, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:13,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:13,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,682 INFO [StoreOpener-4b725c0119f334af117396c8418b7162-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,684 INFO [StoreOpener-4b725c0119f334af117396c8418b7162-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b725c0119f334af117396c8418b7162 columnFamilyName testFamily 2024-11-08T03:12:13,684 DEBUG [StoreOpener-4b725c0119f334af117396c8418b7162-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:13,685 INFO [StoreOpener-4b725c0119f334af117396c8418b7162-1 {}] regionserver.HStore(327): Store=4b725c0119f334af117396c8418b7162/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:13,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,687 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,690 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 4b725c0119f334af117396c8418b7162 2024-11-08T03:12:13,693 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:13,694 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 4b725c0119f334af117396c8418b7162; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62901479, jitterRate=-0.06269492208957672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:13,694 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 4b725c0119f334af117396c8418b7162: 2024-11-08T03:12:13,695 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162., pid=47, masterSystemTime=1731035533674 2024-11-08T03:12:13,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,697 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:12:13,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=4b725c0119f334af117396c8418b7162, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:13,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T03:12:13,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 4b725c0119f334af117396c8418b7162, server=e729c2d9d3a5,45419,1731035408623 in 178 msec 2024-11-08T03:12:13,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-08T03:12:13,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=4b725c0119f334af117396c8418b7162, ASSIGN in 340 msec 2024-11-08T03:12:13,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:13,705 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035533705"}]},"ts":"1731035533705"} 2024-11-08T03:12:13,708 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:12:13,745 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:13,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.2980 sec 2024-11-08T03:12:14,350 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479582 expired clientIPAndPort=172.17.0.2:51384, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:12:14,350 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479583 expired clientIPAndPort=172.17.0.2:51384, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:12:14,350 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479585 expired clientIPAndPort=172.17.0.2:51384, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:12:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T03:12:15,459 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-08T03:12:15,459 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:12:15,460 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:15,467 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T03:12:15,467 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:15,468 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-08T03:12:15,471 DEBUG [hconnection-0x2e9141a0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:15,473 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40302, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:15,481 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:15,485 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4fd11e9c to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d6307c7 2024-11-08T03:12:15,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e72eb88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:15,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:12:15,524 DEBUG [hconnection-0x4b1a86be-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:15,525 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40318, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:15,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:12:15,531 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072082 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:15,532 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:12:15,533 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072082 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:15,533 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T03:12:15,995 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:12:16,871 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:12:18,110 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-08T03:12:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T03:12:18,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:19,216 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:12:19,457 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479576 expired clientIPAndPort=172.17.0.2:51392, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:12:21,543 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T03:12:21.542Z, RpcRetryingCaller{globalStartTime=2024-11-08T03:12:15.532Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T03:12:21.542Z, RpcRetryingCaller{globalStartTime=2024-11-08T03:12:15.532Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:12:21,543 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6011 2024-11-08T03:12:21,544 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4fd11e9c to 127.0.0.1:57399 2024-11-08T03:12:21,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:21,545 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1a2f0601 to 127.0.0.1:57399 2024-11-08T03:12:21,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:21,545 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:21,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:21,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:21,546 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x00a0e159 to 127.0.0.1:57399 2024-11-08T03:12:21,558 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=233 (was 229) - Thread LEAK? -, OpenFileDescriptor=452 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=62 (was 73), ProcessCount=11 (was 11), AvailableMemoryMB=1428 (was 1445) 2024-11-08T03:12:21,566 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=233, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=62, ProcessCount=11, AvailableMemoryMB=1428 2024-11-08T03:12:21,569 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a20bde7 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2bbee81d 2024-11-08T03:12:21,605 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0a20bde7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dfe254c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:21,607 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40534377 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6152a06b 2024-11-08T03:12:21,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e92f623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:21,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T03:12:21,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:21,626 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:21,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-08T03:12:21,628 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T03:12:21,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741863_1039 (size=432) 2024-11-08T03:12:22,034 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:12:22,043 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 577523dc5c55dcc13eabe84cb05dd3b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741864_1040 (size=83) 2024-11-08T03:12:22,458 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:22,459 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 577523dc5c55dcc13eabe84cb05dd3b2, disabling compactions & flushes 2024-11-08T03:12:22,459 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,459 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,459 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. after waiting 0 ms 2024-11-08T03:12:22,459 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,459 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,459 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 577523dc5c55dcc13eabe84cb05dd3b2: 2024-11-08T03:12:22,461 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:22,462 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731035542462"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035542462"}]},"ts":"1731035542462"} 2024-11-08T03:12:22,465 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:22,467 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:22,467 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035542467"}]},"ts":"1731035542467"} 2024-11-08T03:12:22,468 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T03:12:22,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=577523dc5c55dcc13eabe84cb05dd3b2, ASSIGN}] 2024-11-08T03:12:22,513 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=577523dc5c55dcc13eabe84cb05dd3b2, ASSIGN 2024-11-08T03:12:22,515 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=577523dc5c55dcc13eabe84cb05dd3b2, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T03:12:22,666 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=577523dc5c55dcc13eabe84cb05dd3b2, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:22,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 577523dc5c55dcc13eabe84cb05dd3b2, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:22,825 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:22,835 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,836 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => 577523dc5c55dcc13eabe84cb05dd3b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:22,836 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,836 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:22,836 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for 577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,836 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for 577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,839 INFO [StoreOpener-577523dc5c55dcc13eabe84cb05dd3b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,841 INFO [StoreOpener-577523dc5c55dcc13eabe84cb05dd3b2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577523dc5c55dcc13eabe84cb05dd3b2 columnFamilyName testFamily 2024-11-08T03:12:22,842 DEBUG [StoreOpener-577523dc5c55dcc13eabe84cb05dd3b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:22,842 INFO [StoreOpener-577523dc5c55dcc13eabe84cb05dd3b2-1 {}] regionserver.HStore(327): Store=577523dc5c55dcc13eabe84cb05dd3b2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:22,844 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,844 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,848 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for 577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:12:22,851 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:22,852 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened 577523dc5c55dcc13eabe84cb05dd3b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71759557, jitterRate=0.06930072605609894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:22,853 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for 577523dc5c55dcc13eabe84cb05dd3b2: 2024-11-08T03:12:22,853 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., pid=50, masterSystemTime=1731035542825 2024-11-08T03:12:22,855 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,855 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:12:22,856 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=577523dc5c55dcc13eabe84cb05dd3b2, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:22,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-08T03:12:22,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 577523dc5c55dcc13eabe84cb05dd3b2, server=e729c2d9d3a5,45419,1731035408623 in 188 msec 2024-11-08T03:12:22,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T03:12:22,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=577523dc5c55dcc13eabe84cb05dd3b2, ASSIGN in 348 msec 2024-11-08T03:12:22,861 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:22,861 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035542861"}]},"ts":"1731035542861"} 2024-11-08T03:12:22,863 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T03:12:22,924 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:22,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.3010 sec 2024-11-08T03:12:24,064 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479571 expired clientIPAndPort=172.17.0.2:45700, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:12:24,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T03:12:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T03:12:24,637 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-08T03:12:24,637 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T03:12:24,638 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:24,646 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T03:12:24,646 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:24,646 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T03:12:24,649 DEBUG [hconnection-0x655dcaec-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:24,651 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34830, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:24,662 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:24,665 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:12:24,666 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:12:24,667 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0a20bde7 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:12:24,667 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0a20bde7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:24,668 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34838, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:12:24,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:12:25,672 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-08T03:12:25,672 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-08T03:12:25,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:12:26,067 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479569 expired clientIPAndPort=172.17.0.2:45700, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:12:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T03:12:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:12:26,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:12:27,672 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=967ms 2024-11-08T03:12:27,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=967ms 2024-11-08T03:12:27,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:12:27,674 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T03:12:25.673Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-08T03:12:27.673Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=967ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=967ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=967ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T03:12:27,679 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3013 2024-11-08T03:12:27,680 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x40534377 to 127.0.0.1:57399 2024-11-08T03:12:27,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:27,680 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:27,681 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:27,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:27,682 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a20bde7 to 127.0.0.1:57399 2024-11-08T03:12:27,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 233), OpenFileDescriptor=448 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=57 (was 62), ProcessCount=11 (was 11), AvailableMemoryMB=1419 (was 1428) 2024-11-08T03:12:27,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=57, ProcessCount=11, AvailableMemoryMB=1418 2024-11-08T03:12:27,702 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e56dc8c to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@120bb3af 2024-11-08T03:12:27,772 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x3e56dc8c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cff5345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:27,775 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5de30fcb to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31975fa3 2024-11-08T03:12:27,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ad47a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:27,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T03:12:27,793 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:27,793 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:27,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-08T03:12:27,794 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T03:12:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741865_1041 (size=436) 2024-11-08T03:12:28,200 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:12:28,212 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7bb69b6e2882ff2ac3bf9bfd0167d883, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741866_1042 (size=87) 2024-11-08T03:12:28,339 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:12:28,377 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T03:12:28,378 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:28,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:28,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 7bb69b6e2882ff2ac3bf9bfd0167d883, disabling compactions & flushes 2024-11-08T03:12:28,628 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:28,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:28,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. after waiting 0 ms 2024-11-08T03:12:28,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:28,628 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:28,629 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7bb69b6e2882ff2ac3bf9bfd0167d883: 2024-11-08T03:12:28,631 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:28,632 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731035548631"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035548631"}]},"ts":"1731035548631"} 2024-11-08T03:12:28,636 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:28,638 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:28,638 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035548638"}]},"ts":"1731035548638"} 2024-11-08T03:12:28,640 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:12:28,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7bb69b6e2882ff2ac3bf9bfd0167d883, ASSIGN}] 2024-11-08T03:12:28,708 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7bb69b6e2882ff2ac3bf9bfd0167d883, ASSIGN 2024-11-08T03:12:28,710 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7bb69b6e2882ff2ac3bf9bfd0167d883, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T03:12:28,861 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=7bb69b6e2882ff2ac3bf9bfd0167d883, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:28,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 7bb69b6e2882ff2ac3bf9bfd0167d883, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:29,015 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:29,027 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:29,028 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => 7bb69b6e2882ff2ac3bf9bfd0167d883, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:29,029 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,029 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:29,029 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for 7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,029 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for 7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,031 INFO [StoreOpener-7bb69b6e2882ff2ac3bf9bfd0167d883-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,033 INFO [StoreOpener-7bb69b6e2882ff2ac3bf9bfd0167d883-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bb69b6e2882ff2ac3bf9bfd0167d883 columnFamilyName testFamily 2024-11-08T03:12:29,033 DEBUG [StoreOpener-7bb69b6e2882ff2ac3bf9bfd0167d883-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:29,033 INFO [StoreOpener-7bb69b6e2882ff2ac3bf9bfd0167d883-1 {}] regionserver.HStore(327): Store=7bb69b6e2882ff2ac3bf9bfd0167d883/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:29,034 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,035 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,037 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for 7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:12:29,040 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:29,040 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened 7bb69b6e2882ff2ac3bf9bfd0167d883; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64136867, jitterRate=-0.04428620636463165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:29,041 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for 7bb69b6e2882ff2ac3bf9bfd0167d883: 2024-11-08T03:12:29,042 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883., pid=53, masterSystemTime=1731035549015 2024-11-08T03:12:29,044 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:29,044 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:12:29,045 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=7bb69b6e2882ff2ac3bf9bfd0167d883, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:29,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T03:12:29,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 7bb69b6e2882ff2ac3bf9bfd0167d883, server=e729c2d9d3a5,45419,1731035408623 in 185 msec 2024-11-08T03:12:29,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-08T03:12:29,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7bb69b6e2882ff2ac3bf9bfd0167d883, ASSIGN in 345 msec 2024-11-08T03:12:29,052 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:29,052 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035549052"}]},"ts":"1731035549052"} 2024-11-08T03:12:29,054 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:12:29,062 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:29,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.2720 sec 2024-11-08T03:12:30,117 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-08T03:12:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T03:12:30,805 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-08T03:12:30,805 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:12:30,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:30,812 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T03:12:30,812 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:30,813 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-08T03:12:30,816 DEBUG [hconnection-0x6b0e3ad4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:30,818 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46732, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:30,826 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:30,828 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T03:12:30,828 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:12:30,830 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x3e56dc8c {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:12:30,830 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x3e56dc8c {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:30,831 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46744, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:12:30,834 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T03:12:30,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072094 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:30,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T03:12:30,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072094 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:30,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479522 2024-11-08T03:12:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46744 deadline: 1731035553834, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:30,836 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072094 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:12:30,837 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T03:12:30,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072095 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:30,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T03:12:30,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:30,839 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:30,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072096 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:30,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479520 2024-11-08T03:12:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46744 deadline: 1731035553838, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:30,840 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072096 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:12:30,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:30,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072097 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:30,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-279807653226479519 2024-11-08T03:12:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46744 deadline: 1731035553840, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:30,842 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072097 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:12:30,843 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:30,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072098 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:30,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-279807653226479518 2024-11-08T03:12:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46744 deadline: 1731035553842, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:30,844 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072098 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:12:30,845 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:30,846 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5de30fcb to 127.0.0.1:57399 2024-11-08T03:12:30,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:30,846 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:30,846 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:30,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:30,846 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3e56dc8c to 127.0.0.1:57399 2024-11-08T03:12:30,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=52 (was 57), ProcessCount=11 (was 11), AvailableMemoryMB=1399 (was 1418) 2024-11-08T03:12:30,862 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=52, ProcessCount=11, AvailableMemoryMB=1398 2024-11-08T03:12:30,863 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24dbfdae to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1683a517 2024-11-08T03:12:30,896 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x24dbfdae {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72704fb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:30,897 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43a4445b to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d801b0 2024-11-08T03:12:30,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c95aa68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:30,908 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:30,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T03:12:30,910 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:30,910 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:30,911 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:30,912 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-08T03:12:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T03:12:30,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741867_1043 (size=422) 2024-11-08T03:12:31,328 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ff1ef2a898f94dec827a806db3b081f4, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741868_1044 (size=73) 2024-11-08T03:12:31,742 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:31,742 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing ff1ef2a898f94dec827a806db3b081f4, disabling compactions & flushes 2024-11-08T03:12:31,743 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:31,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:31,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. after waiting 0 ms 2024-11-08T03:12:31,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:31,743 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:31,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for ff1ef2a898f94dec827a806db3b081f4: 2024-11-08T03:12:31,745 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:31,746 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731035551745"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035551745"}]},"ts":"1731035551745"} 2024-11-08T03:12:31,748 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:31,749 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:31,749 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035551749"}]},"ts":"1731035551749"} 2024-11-08T03:12:31,751 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:12:31,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ff1ef2a898f94dec827a806db3b081f4, ASSIGN}] 2024-11-08T03:12:31,796 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ff1ef2a898f94dec827a806db3b081f4, ASSIGN 2024-11-08T03:12:31,797 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ff1ef2a898f94dec827a806db3b081f4, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T03:12:31,948 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ff1ef2a898f94dec827a806db3b081f4, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:31,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure ff1ef2a898f94dec827a806db3b081f4, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:32,105 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:32,115 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:32,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => ff1ef2a898f94dec827a806db3b081f4, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:32,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:32,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,119 INFO [StoreOpener-ff1ef2a898f94dec827a806db3b081f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,121 INFO [StoreOpener-ff1ef2a898f94dec827a806db3b081f4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff1ef2a898f94dec827a806db3b081f4 columnFamilyName testFamily 2024-11-08T03:12:32,121 DEBUG [StoreOpener-ff1ef2a898f94dec827a806db3b081f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:32,122 INFO [StoreOpener-ff1ef2a898f94dec827a806db3b081f4-1 {}] regionserver.HStore(327): Store=ff1ef2a898f94dec827a806db3b081f4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:32,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,124 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:12:32,131 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:32,132 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened ff1ef2a898f94dec827a806db3b081f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74484044, jitterRate=0.10989874601364136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:32,134 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for ff1ef2a898f94dec827a806db3b081f4: 2024-11-08T03:12:32,135 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4., pid=56, masterSystemTime=1731035552105 2024-11-08T03:12:32,137 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:32,137 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:12:32,137 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ff1ef2a898f94dec827a806db3b081f4, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:32,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T03:12:32,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure ff1ef2a898f94dec827a806db3b081f4, server=e729c2d9d3a5,45419,1731035408623 in 187 msec 2024-11-08T03:12:32,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T03:12:32,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ff1ef2a898f94dec827a806db3b081f4, ASSIGN in 346 msec 2024-11-08T03:12:32,142 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:32,142 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035552142"}]},"ts":"1731035552142"} 2024-11-08T03:12:32,143 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:12:32,153 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:32,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.2450 sec 2024-11-08T03:12:32,176 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479564 expired clientIPAndPort=172.17.0.2:39442, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:12:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T03:12:33,918 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-08T03:12:33,919 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:12:33,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:33,927 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T03:12:33,927 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:33,928 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T03:12:33,930 DEBUG [hconnection-0x576780-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:33,932 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46760, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:33,939 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:33,941 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:12:33,942 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:12:34,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:12:36,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-08T03:12:36,938 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:12:38,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T03:12:38,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:38,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T03:12:38,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T03:12:40,969 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:12:42,952 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11026: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11026: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:12:42,953 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T03:12:42.953Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11026: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11026: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1984ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:12:42,954 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9012 2024-11-08T03:12:42,955 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x43a4445b to 127.0.0.1:57399 2024-11-08T03:12:42,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:42,955 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:42,955 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:42,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:42,956 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x24dbfdae to 127.0.0.1:57399 2024-11-08T03:12:42,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=228 (was 233), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=44 (was 52), ProcessCount=11 (was 11), AvailableMemoryMB=1349 (was 1398) 2024-11-08T03:12:42,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=228, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=44, ProcessCount=11, AvailableMemoryMB=1349 2024-11-08T03:12:42,975 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2dfcba39 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4549c99b 2024-11-08T03:12:42,987 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x2dfcba39 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0ece5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:42,988 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x33ddf85a to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3318ab91 2024-11-08T03:12:42,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b7de2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:42,998 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:42,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T03:12:43,000 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:43,000 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:43,000 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-08T03:12:43,001 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T03:12:43,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741869_1045 (size=420) 2024-11-08T03:12:43,416 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d6de0e77e271698fa27c712288e77738, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:43,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741870_1046 (size=71) 2024-11-08T03:12:43,697 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:12:43,832 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:43,833 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing d6de0e77e271698fa27c712288e77738, disabling compactions & flushes 2024-11-08T03:12:43,833 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:43,833 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:43,833 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. after waiting 0 ms 2024-11-08T03:12:43,833 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:43,833 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:43,833 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for d6de0e77e271698fa27c712288e77738: 2024-11-08T03:12:43,835 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:43,836 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731035563835"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035563835"}]},"ts":"1731035563835"} 2024-11-08T03:12:43,838 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:43,839 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:43,840 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035563839"}]},"ts":"1731035563839"} 2024-11-08T03:12:43,842 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:12:43,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d6de0e77e271698fa27c712288e77738, ASSIGN}] 2024-11-08T03:12:43,887 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d6de0e77e271698fa27c712288e77738, ASSIGN 2024-11-08T03:12:43,888 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d6de0e77e271698fa27c712288e77738, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T03:12:44,039 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=d6de0e77e271698fa27c712288e77738, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:44,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure d6de0e77e271698fa27c712288e77738, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:44,194 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:44,207 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:44,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => d6de0e77e271698fa27c712288e77738, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:44,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:44,209 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,209 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,212 INFO [StoreOpener-d6de0e77e271698fa27c712288e77738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,215 INFO [StoreOpener-d6de0e77e271698fa27c712288e77738-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6de0e77e271698fa27c712288e77738 columnFamilyName testFamily 2024-11-08T03:12:44,215 DEBUG [StoreOpener-d6de0e77e271698fa27c712288e77738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:44,216 INFO [StoreOpener-d6de0e77e271698fa27c712288e77738-1 {}] regionserver.HStore(327): Store=d6de0e77e271698fa27c712288e77738/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:44,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,220 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for d6de0e77e271698fa27c712288e77738 2024-11-08T03:12:44,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:44,223 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened d6de0e77e271698fa27c712288e77738; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70468803, jitterRate=0.0500669926404953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:44,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for d6de0e77e271698fa27c712288e77738: 2024-11-08T03:12:44,225 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738., pid=59, masterSystemTime=1731035564194 2024-11-08T03:12:44,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:44,227 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:12:44,227 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=d6de0e77e271698fa27c712288e77738, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:44,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T03:12:44,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure d6de0e77e271698fa27c712288e77738, server=e729c2d9d3a5,45419,1731035408623 in 188 msec 2024-11-08T03:12:44,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T03:12:44,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d6de0e77e271698fa27c712288e77738, ASSIGN in 344 msec 2024-11-08T03:12:44,232 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:44,233 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035564232"}]},"ts":"1731035564232"} 2024-11-08T03:12:44,234 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:12:44,252 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:44,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 1.2540 sec 2024-11-08T03:12:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T03:12:46,009 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-08T03:12:46,009 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:12:46,010 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:46,019 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T03:12:46,019 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:46,019 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T03:12:46,022 DEBUG [hconnection-0x4efefbd7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:46,024 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50300, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:46,032 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:46,034 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:12:46,034 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:12:46,036 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x2dfcba39 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:12:46,036 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x2dfcba39 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:46,037 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50302, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:12:46,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:12:46,039 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072110 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:46,040 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:12:46,040 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072110 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:46,041 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T03:12:47,470 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:12:48,122 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-08T03:12:48,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T03:12:48,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:49,751 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:12:50,601 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479551 expired clientIPAndPort=172.17.0.2:56362, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:12:51,636 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:12:55,043 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9002 2024-11-08T03:12:55,044 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x33ddf85a to 127.0.0.1:57399 2024-11-08T03:12:55,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:55,044 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:55,044 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072110 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:12:55,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:55,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:55,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479506' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T03:12:55,046 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2dfcba39 to 127.0.0.1:57399 2024-11-08T03:12:55,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:50302 deadline: 1731035581043, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479506' does not exist 2024-11-08T03:12:55,052 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:12:55,046 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e729c2d9d3a5,45419,1731035408623 for closing scanner id = -279807653226479506 for d6de0e77e271698fa27c712288e77738 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T03:12:55,058 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227 (was 228), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=37 (was 44), ProcessCount=11 (was 11), AvailableMemoryMB=1309 (was 1349) 2024-11-08T03:12:55,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=37, ProcessCount=11, AvailableMemoryMB=1309 2024-11-08T03:12:55,070 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05c4adcf to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42b8448f 2024-11-08T03:12:55,104 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x05c4adcf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193cdb54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:55,106 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1e947698 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3201081b 2024-11-08T03:12:55,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128b6141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:55,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T03:12:55,127 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:55,128 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:55,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-08T03:12:55,129 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T03:12:55,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741871_1047 (size=431) 2024-11-08T03:12:55,542 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:12:55,548 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0b82769fbf29b51a6e6507fe4c11db8d, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741872_1048 (size=82) 2024-11-08T03:12:55,969 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:55,969 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 0b82769fbf29b51a6e6507fe4c11db8d, disabling compactions & flushes 2024-11-08T03:12:55,969 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:55,969 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:55,969 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. after waiting 0 ms 2024-11-08T03:12:55,969 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:55,969 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:55,969 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0b82769fbf29b51a6e6507fe4c11db8d: 2024-11-08T03:12:55,970 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:55,971 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731035575970"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035575970"}]},"ts":"1731035575970"} 2024-11-08T03:12:55,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:55,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:55,974 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035575973"}]},"ts":"1731035575973"} 2024-11-08T03:12:55,975 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:12:56,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=0b82769fbf29b51a6e6507fe4c11db8d, ASSIGN}] 2024-11-08T03:12:56,021 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=0b82769fbf29b51a6e6507fe4c11db8d, ASSIGN 2024-11-08T03:12:56,022 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=0b82769fbf29b51a6e6507fe4c11db8d, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T03:12:56,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=0b82769fbf29b51a6e6507fe4c11db8d, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:56,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 0b82769fbf29b51a6e6507fe4c11db8d, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:56,327 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:56,339 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:56,339 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 0b82769fbf29b51a6e6507fe4c11db8d, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:56,340 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,340 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:56,340 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,340 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,342 INFO [StoreOpener-0b82769fbf29b51a6e6507fe4c11db8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,344 INFO [StoreOpener-0b82769fbf29b51a6e6507fe4c11db8d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b82769fbf29b51a6e6507fe4c11db8d columnFamilyName testFamily 2024-11-08T03:12:56,344 DEBUG [StoreOpener-0b82769fbf29b51a6e6507fe4c11db8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:56,345 INFO [StoreOpener-0b82769fbf29b51a6e6507fe4c11db8d-1 {}] regionserver.HStore(327): Store=0b82769fbf29b51a6e6507fe4c11db8d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:56,346 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,347 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,350 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:12:56,354 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:56,354 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 0b82769fbf29b51a6e6507fe4c11db8d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61157334, jitterRate=-0.08868470788002014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:56,356 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 0b82769fbf29b51a6e6507fe4c11db8d: 2024-11-08T03:12:56,357 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d., pid=62, masterSystemTime=1731035576327 2024-11-08T03:12:56,359 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:56,359 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:12:56,359 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=0b82769fbf29b51a6e6507fe4c11db8d, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:56,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-08T03:12:56,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 0b82769fbf29b51a6e6507fe4c11db8d, server=e729c2d9d3a5,45419,1731035408623 in 186 msec 2024-11-08T03:12:56,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T03:12:56,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=0b82769fbf29b51a6e6507fe4c11db8d, ASSIGN in 343 msec 2024-11-08T03:12:56,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:56,365 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035576365"}]},"ts":"1731035576365"} 2024-11-08T03:12:56,366 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:12:56,377 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:56,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.2530 sec 2024-11-08T03:12:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T03:12:58,141 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-08T03:12:58,141 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:12:58,142 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:58,154 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-08T03:12:58,154 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:12:58,154 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-08T03:12:58,156 DEBUG [hconnection-0x5cb92963-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:12:58,157 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38116, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:12:58,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:12:58,167 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T03:12:58,168 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T03:12:58,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072114 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:58,170 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T03:12:58,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072114 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:58,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479502 2024-11-08T03:12:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38116 deadline: 1731035581170, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:58,171 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:12:58,173 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T03:12:58,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072115 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:58,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T03:12:58,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:58,176 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:58,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072116 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:58,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-279807653226479500 2024-11-08T03:12:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38116 deadline: 1731035581176, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:58,178 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:12:58,179 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:58,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072117 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:58,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-279807653226479499 2024-11-08T03:12:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38116 deadline: 1731035581180, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:58,181 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:12:58,183 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:58,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072118 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:12:58,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-279807653226479498 2024-11-08T03:12:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38116 deadline: 1731035581183, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T03:12:58,185 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:12:58,187 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T03:12:58,187 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1e947698 to 127.0.0.1:57399 2024-11-08T03:12:58,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:58,188 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:12:58,188 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:12:58,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:12:58,188 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x05c4adcf to 127.0.0.1:57399 2024-11-08T03:12:58,196 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=234 (was 227) - Thread LEAK? -, OpenFileDescriptor=450 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=34 (was 37), ProcessCount=11 (was 11), AvailableMemoryMB=1299 (was 1309) 2024-11-08T03:12:58,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=34, ProcessCount=11, AvailableMemoryMB=1298 2024-11-08T03:12:58,207 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62ee8de9 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b392d37 2024-11-08T03:12:58,238 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x62ee8de9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a7709a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:58,240 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b40394f to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a09a254 2024-11-08T03:12:58,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T03:12:58,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:12:58,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1ef939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:12:58,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:12:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T03:12:58,260 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:12:58,260 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:58,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-08T03:12:58,261 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:12:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T03:12:58,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741873_1049 (size=425) 2024-11-08T03:12:58,679 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 19566d9f13bc4850b9dbe183d03f6f25, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:12:58,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741874_1050 (size=76) 2024-11-08T03:12:59,089 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:59,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 19566d9f13bc4850b9dbe183d03f6f25, disabling compactions & flushes 2024-11-08T03:12:59,090 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. after waiting 0 ms 2024-11-08T03:12:59,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,090 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,091 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 19566d9f13bc4850b9dbe183d03f6f25: 2024-11-08T03:12:59,093 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:12:59,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731035579094"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035579094"}]},"ts":"1731035579094"} 2024-11-08T03:12:59,098 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:12:59,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:12:59,099 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035579099"}]},"ts":"1731035579099"} 2024-11-08T03:12:59,101 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:12:59,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=19566d9f13bc4850b9dbe183d03f6f25, ASSIGN}] 2024-11-08T03:12:59,154 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=19566d9f13bc4850b9dbe183d03f6f25, ASSIGN 2024-11-08T03:12:59,156 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=19566d9f13bc4850b9dbe183d03f6f25, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:12:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T03:12:59,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=19566d9f13bc4850b9dbe183d03f6f25, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:59,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure 19566d9f13bc4850b9dbe183d03f6f25, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:12:59,460 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:59,467 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,467 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => 19566d9f13bc4850b9dbe183d03f6f25, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:12:59,468 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true 19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,468 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:12:59,468 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for 19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,468 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for 19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,471 INFO [StoreOpener-19566d9f13bc4850b9dbe183d03f6f25-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,474 INFO [StoreOpener-19566d9f13bc4850b9dbe183d03f6f25-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19566d9f13bc4850b9dbe183d03f6f25 columnFamilyName testFamily 2024-11-08T03:12:59,474 DEBUG [StoreOpener-19566d9f13bc4850b9dbe183d03f6f25-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:12:59,475 INFO [StoreOpener-19566d9f13bc4850b9dbe183d03f6f25-1 {}] regionserver.HStore(327): Store=19566d9f13bc4850b9dbe183d03f6f25/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:12:59,476 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,476 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,479 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for 19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:12:59,481 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:12:59,481 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened 19566d9f13bc4850b9dbe183d03f6f25; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72231233, jitterRate=0.07632924616336823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:12:59,482 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for 19566d9f13bc4850b9dbe183d03f6f25: 2024-11-08T03:12:59,483 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25., pid=65, masterSystemTime=1731035579460 2024-11-08T03:12:59,484 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,485 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:12:59,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=19566d9f13bc4850b9dbe183d03f6f25, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:12:59,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T03:12:59,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure 19566d9f13bc4850b9dbe183d03f6f25, server=e729c2d9d3a5,45419,1731035408623 in 178 msec 2024-11-08T03:12:59,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-08T03:12:59,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=19566d9f13bc4850b9dbe183d03f6f25, ASSIGN in 335 msec 2024-11-08T03:12:59,490 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:12:59,490 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035579490"}]},"ts":"1731035579490"} 2024-11-08T03:12:59,491 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:12:59,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:12:59,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.2440 sec 2024-11-08T03:13:00,127 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-08T03:13:00,129 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-08T03:13:00,315 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479547 expired clientIPAndPort=172.17.0.2:56376, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:13:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T03:13:01,270 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-08T03:13:01,270 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:01,270 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:01,282 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:01,282 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:01,282 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T03:13:01,285 DEBUG [hconnection-0x48ac0a60-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:01,286 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38122, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:01,297 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:01,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:01,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:13:01,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T03:13:01,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T03:13:03,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:13:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38122 deadline: 1731035584302, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:13:03,317 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:13:03,318 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:13:03,319 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2019 2024-11-08T03:13:03,319 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b40394f to 127.0.0.1:57399 2024-11-08T03:13:03,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:03,319 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:03,320 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:03,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:03,320 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62ee8de9 to 127.0.0.1:57399 2024-11-08T03:13:03,330 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233 (was 234), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=31 (was 34), ProcessCount=11 (was 11), AvailableMemoryMB=1263 (was 1298) 2024-11-08T03:13:03,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=31, ProcessCount=11, AvailableMemoryMB=1263 2024-11-08T03:13:03,340 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08afcff5 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c91eb01 2024-11-08T03:13:03,362 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x08afcff5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19eeb258, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:03,363 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21f6a47c to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c5a1257 2024-11-08T03:13:03,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afca24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:03,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T03:13:03,378 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:03,378 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:03,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-08T03:13:03,379 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T03:13:03,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741875_1051 (size=426) 2024-11-08T03:13:03,793 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 01e439eb5eab7a7ae47039d307f0cfd0, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:03,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741876_1052 (size=77) 2024-11-08T03:13:04,204 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:04,204 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 01e439eb5eab7a7ae47039d307f0cfd0, disabling compactions & flushes 2024-11-08T03:13:04,204 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,204 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,204 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. after waiting 0 ms 2024-11-08T03:13:04,205 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,205 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,205 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 01e439eb5eab7a7ae47039d307f0cfd0: 2024-11-08T03:13:04,207 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:04,207 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731035584207"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035584207"}]},"ts":"1731035584207"} 2024-11-08T03:13:04,209 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:04,210 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:04,210 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035584210"}]},"ts":"1731035584210"} 2024-11-08T03:13:04,212 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:04,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=01e439eb5eab7a7ae47039d307f0cfd0, ASSIGN}] 2024-11-08T03:13:04,264 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=01e439eb5eab7a7ae47039d307f0cfd0, ASSIGN 2024-11-08T03:13:04,266 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=01e439eb5eab7a7ae47039d307f0cfd0, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T03:13:04,417 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=01e439eb5eab7a7ae47039d307f0cfd0, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:04,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 01e439eb5eab7a7ae47039d307f0cfd0, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:04,577 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:04,587 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,587 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 01e439eb5eab7a7ae47039d307f0cfd0, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:04,587 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,588 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:04,588 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,588 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,589 INFO [StoreOpener-01e439eb5eab7a7ae47039d307f0cfd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,591 INFO [StoreOpener-01e439eb5eab7a7ae47039d307f0cfd0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e439eb5eab7a7ae47039d307f0cfd0 columnFamilyName testFamily 2024-11-08T03:13:04,591 DEBUG [StoreOpener-01e439eb5eab7a7ae47039d307f0cfd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:04,592 INFO [StoreOpener-01e439eb5eab7a7ae47039d307f0cfd0-1 {}] regionserver.HStore(327): Store=01e439eb5eab7a7ae47039d307f0cfd0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:04,593 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,593 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,595 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:13:04,598 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:04,598 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 01e439eb5eab7a7ae47039d307f0cfd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63114279, jitterRate=-0.05952395498752594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:04,599 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 01e439eb5eab7a7ae47039d307f0cfd0: 2024-11-08T03:13:04,600 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., pid=68, masterSystemTime=1731035584577 2024-11-08T03:13:04,601 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,601 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:13:04,602 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=01e439eb5eab7a7ae47039d307f0cfd0, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:04,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-08T03:13:04,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 01e439eb5eab7a7ae47039d307f0cfd0, server=e729c2d9d3a5,45419,1731035408623 in 181 msec 2024-11-08T03:13:04,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T03:13:04,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=01e439eb5eab7a7ae47039d307f0cfd0, ASSIGN in 342 msec 2024-11-08T03:13:04,606 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:04,606 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035584606"}]},"ts":"1731035584606"} 2024-11-08T03:13:04,607 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:04,618 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:04,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.2440 sec 2024-11-08T03:13:06,131 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-08T03:13:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T03:13:06,388 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-08T03:13:06,388 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:06,389 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:06,399 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:06,400 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:06,400 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T03:13:06,402 DEBUG [hconnection-0x1e925a08-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:06,403 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38126, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:06,410 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:06,412 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:13:06,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:13:06,938 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:13:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T03:13:08,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T03:13:08,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:08,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T03:13:08,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:08,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:13:09,422 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5016: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5016: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:13:09,423 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T03:13:09.423Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5016: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5016: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:13:09,425 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3012 2024-11-08T03:13:09,426 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21f6a47c to 127.0.0.1:57399 2024-11-08T03:13:09,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:09,426 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:09,426 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:09,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:09,427 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08afcff5 to 127.0.0.1:57399 2024-11-08T03:13:09,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=37 (was 31) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1252 (was 1263) 2024-11-08T03:13:09,447 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=37, ProcessCount=11, AvailableMemoryMB=1251 2024-11-08T03:13:09,449 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x108ccfc5 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5fba8d2b 2024-11-08T03:13:09,471 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x108ccfc5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719e9fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:09,472 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01858bfd to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c356263 2024-11-08T03:13:09,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3164c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:09,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T03:13:09,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:09,494 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:09,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-08T03:13:09,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T03:13:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741877_1053 (size=424) 2024-11-08T03:13:09,908 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 80ca2120f66e2dbe59de9827fecc552f, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:09,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741878_1054 (size=75) 2024-11-08T03:13:09,930 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479541 expired clientIPAndPort=172.17.0.2:55022, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:13:09,937 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:13:10,091 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:10,317 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:10,317 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 80ca2120f66e2dbe59de9827fecc552f, disabling compactions & flushes 2024-11-08T03:13:10,317 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,317 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,317 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. after waiting 0 ms 2024-11-08T03:13:10,317 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,317 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,317 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 80ca2120f66e2dbe59de9827fecc552f: 2024-11-08T03:13:10,318 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:10,318 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731035590318"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035590318"}]},"ts":"1731035590318"} 2024-11-08T03:13:10,320 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:10,320 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:10,320 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035590320"}]},"ts":"1731035590320"} 2024-11-08T03:13:10,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:10,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=80ca2120f66e2dbe59de9827fecc552f, ASSIGN}] 2024-11-08T03:13:10,375 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=80ca2120f66e2dbe59de9827fecc552f, ASSIGN 2024-11-08T03:13:10,376 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=80ca2120f66e2dbe59de9827fecc552f, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T03:13:10,527 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=80ca2120f66e2dbe59de9827fecc552f, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:10,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure 80ca2120f66e2dbe59de9827fecc552f, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:10,681 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:10,693 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,693 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => 80ca2120f66e2dbe59de9827fecc552f, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:10,694 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true 80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,694 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:10,694 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for 80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,694 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for 80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,696 INFO [StoreOpener-80ca2120f66e2dbe59de9827fecc552f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,698 INFO [StoreOpener-80ca2120f66e2dbe59de9827fecc552f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80ca2120f66e2dbe59de9827fecc552f columnFamilyName testFamily 2024-11-08T03:13:10,698 DEBUG [StoreOpener-80ca2120f66e2dbe59de9827fecc552f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:10,699 INFO [StoreOpener-80ca2120f66e2dbe59de9827fecc552f-1 {}] regionserver.HStore(327): Store=80ca2120f66e2dbe59de9827fecc552f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:10,700 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,700 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,703 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for 80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:13:10,706 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:10,707 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened 80ca2120f66e2dbe59de9827fecc552f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71362223, jitterRate=0.06337998807430267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:10,708 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for 80ca2120f66e2dbe59de9827fecc552f: 2024-11-08T03:13:10,709 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f., pid=71, masterSystemTime=1731035590681 2024-11-08T03:13:10,711 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,711 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:13:10,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=80ca2120f66e2dbe59de9827fecc552f, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:10,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T03:13:10,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure 80ca2120f66e2dbe59de9827fecc552f, server=e729c2d9d3a5,45419,1731035408623 in 185 msec 2024-11-08T03:13:10,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-08T03:13:10,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=80ca2120f66e2dbe59de9827fecc552f, ASSIGN in 342 msec 2024-11-08T03:13:10,718 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:10,718 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035590718"}]},"ts":"1731035590718"} 2024-11-08T03:13:10,720 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:10,779 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:10,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.2890 sec 2024-11-08T03:13:12,132 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-08T03:13:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T03:13:12,500 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-08T03:13:12,500 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:12,500 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:12,504 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:12,505 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:12,505 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T03:13:12,507 DEBUG [hconnection-0x1af9cbbc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:12,508 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45888, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:12,515 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:12,517 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:12,518 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:13:12,519 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x108ccfc5 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:13:12,519 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x108ccfc5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:12,520 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45892, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:13:12,522 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:13:12,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072135 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:12,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:13:12,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072135 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:12,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T03:13:14,226 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:13:15,533 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3009 2024-11-08T03:13:15,533 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01858bfd to 127.0.0.1:57399 2024-11-08T03:13:15,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:15,533 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:15,533 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072135 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:13:15,534 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:15,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:15,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479481' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T03:13:15,535 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x108ccfc5 to 127.0.0.1:57399 2024-11-08T03:13:15,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:45892 deadline: 1731035596533, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-279807653226479481' does not exist 2024-11-08T03:13:15,540 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:13:15,535 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e729c2d9d3a5,45419,1731035408623 for closing scanner id = -279807653226479481 for 80ca2120f66e2dbe59de9827fecc552f of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T03:13:15,549 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=229 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=31 (was 37), ProcessCount=11 (was 11), AvailableMemoryMB=1247 (was 1251) 2024-11-08T03:13:15,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=31, ProcessCount=11, AvailableMemoryMB=1247 2024-11-08T03:13:15,559 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x15b2237d to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b8a930e 2024-11-08T03:13:15,570 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x15b2237d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af007b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:15,571 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7165d507 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f0cf1cf 2024-11-08T03:13:15,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba18a98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:15,589 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:15,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-08T03:13:15,592 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:15,592 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:15,592 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-08T03:13:15,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T03:13:15,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741879_1055 (size=416) 2024-11-08T03:13:16,009 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 565426ad1c8cfb593f23db513d128881, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741880_1056 (size=67) 2024-11-08T03:13:16,024 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:13:16,197 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:16,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:16,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 565426ad1c8cfb593f23db513d128881, disabling compactions & flushes 2024-11-08T03:13:16,423 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. after waiting 0 ms 2024-11-08T03:13:16,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,424 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 565426ad1c8cfb593f23db513d128881: 2024-11-08T03:13:16,425 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:16,426 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731035596425"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035596425"}]},"ts":"1731035596425"} 2024-11-08T03:13:16,427 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:16,428 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:16,428 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035596428"}]},"ts":"1731035596428"} 2024-11-08T03:13:16,430 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:16,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=565426ad1c8cfb593f23db513d128881, ASSIGN}] 2024-11-08T03:13:16,451 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=565426ad1c8cfb593f23db513d128881, ASSIGN 2024-11-08T03:13:16,452 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=565426ad1c8cfb593f23db513d128881, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T03:13:16,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=565426ad1c8cfb593f23db513d128881, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:16,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 565426ad1c8cfb593f23db513d128881, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:16,755 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:16,761 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 565426ad1c8cfb593f23db513d128881, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:16,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:16,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,765 INFO [StoreOpener-565426ad1c8cfb593f23db513d128881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,768 INFO [StoreOpener-565426ad1c8cfb593f23db513d128881-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 565426ad1c8cfb593f23db513d128881 columnFamilyName testFamily 2024-11-08T03:13:16,769 DEBUG [StoreOpener-565426ad1c8cfb593f23db513d128881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:16,770 INFO [StoreOpener-565426ad1c8cfb593f23db513d128881-1 {}] regionserver.HStore(327): Store=565426ad1c8cfb593f23db513d128881/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:16,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 565426ad1c8cfb593f23db513d128881 2024-11-08T03:13:16,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:16,777 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 565426ad1c8cfb593f23db513d128881; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68466274, jitterRate=0.020226985216140747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:16,778 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 565426ad1c8cfb593f23db513d128881: 2024-11-08T03:13:16,778 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881., pid=74, masterSystemTime=1731035596755 2024-11-08T03:13:16,780 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,780 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:13:16,780 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=565426ad1c8cfb593f23db513d128881, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:16,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-08T03:13:16,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 565426ad1c8cfb593f23db513d128881, server=e729c2d9d3a5,45419,1731035408623 in 178 msec 2024-11-08T03:13:16,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T03:13:16,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=565426ad1c8cfb593f23db513d128881, ASSIGN in 333 msec 2024-11-08T03:13:16,785 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:16,785 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035596785"}]},"ts":"1731035596785"} 2024-11-08T03:13:16,786 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:16,801 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:16,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.2120 sec 2024-11-08T03:13:16,942 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479539 expired clientIPAndPort=172.17.0.2:55022, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:13:18,133 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-08T03:13:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T03:13:18,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T03:13:18,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T03:13:18,598 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-08T03:13:18,598 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:18,599 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:18,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-08T03:13:18,605 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072136 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:18,607 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:18,607 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:18,607 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-08T03:13:18,610 DEBUG [hconnection-0x533da300-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:18,611 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36448, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:18,620 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:18,622 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:18,624 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072139 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:13:18,624 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T03:13:22,047 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479534 expired clientIPAndPort=172.17.0.2:40318, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:13:22,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:25,125 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-08T03:13:25,125 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7165d507 to 127.0.0.1:57399 2024-11-08T03:13:25,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:25,125 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:25,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:25,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:25,125 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x15b2237d to 127.0.0.1:57399 2024-11-08T03:13:25,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230 (was 229) - Thread LEAK? -, OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=36 (was 31) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2093 (was 1247) - AvailableMemoryMB LEAK? - 2024-11-08T03:13:25,140 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:13:25,142 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=36, ProcessCount=11, AvailableMemoryMB=2092 2024-11-08T03:13:25,144 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7c34f73b to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8f0e554 2024-11-08T03:13:25,170 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x7c34f73b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626da311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:25,171 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c503948 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e4b4de 2024-11-08T03:13:25,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3216d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:25,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-08T03:13:25,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:25,188 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:25,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-08T03:13:25,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T03:13:25,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741881_1057 (size=420) 2024-11-08T03:13:25,601 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c3ba4cef1335634802f30cc1514d1293, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:25,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741882_1058 (size=71) 2024-11-08T03:13:25,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:25,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing c3ba4cef1335634802f30cc1514d1293, disabling compactions & flushes 2024-11-08T03:13:25,615 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. after waiting 0 ms 2024-11-08T03:13:25,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,615 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for c3ba4cef1335634802f30cc1514d1293: 2024-11-08T03:13:25,617 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:25,617 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731035605617"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035605617"}]},"ts":"1731035605617"} 2024-11-08T03:13:25,619 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:25,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:25,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035605621"}]},"ts":"1731035605621"} 2024-11-08T03:13:25,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:25,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=c3ba4cef1335634802f30cc1514d1293, ASSIGN}] 2024-11-08T03:13:25,643 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=c3ba4cef1335634802f30cc1514d1293, ASSIGN 2024-11-08T03:13:25,644 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=c3ba4cef1335634802f30cc1514d1293, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:25,794 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=c3ba4cef1335634802f30cc1514d1293, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:25,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure c3ba4cef1335634802f30cc1514d1293, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:25,949 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:25,954 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => c3ba4cef1335634802f30cc1514d1293, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,956 INFO [StoreOpener-c3ba4cef1335634802f30cc1514d1293-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,958 INFO [StoreOpener-c3ba4cef1335634802f30cc1514d1293-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ba4cef1335634802f30cc1514d1293 columnFamilyName testFamily 2024-11-08T03:13:25,958 DEBUG [StoreOpener-c3ba4cef1335634802f30cc1514d1293-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:25,958 INFO [StoreOpener-c3ba4cef1335634802f30cc1514d1293-1 {}] regionserver.HStore(327): Store=c3ba4cef1335634802f30cc1514d1293/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:25,959 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,960 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,962 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:13:25,965 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:25,966 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened c3ba4cef1335634802f30cc1514d1293; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65333183, jitterRate=-0.0264597088098526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:25,967 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for c3ba4cef1335634802f30cc1514d1293: 2024-11-08T03:13:25,968 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293., pid=77, masterSystemTime=1731035605949 2024-11-08T03:13:25,970 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,970 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:13:25,970 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=c3ba4cef1335634802f30cc1514d1293, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:25,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T03:13:25,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure c3ba4cef1335634802f30cc1514d1293, server=e729c2d9d3a5,45419,1731035408623 in 176 msec 2024-11-08T03:13:25,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-08T03:13:25,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=c3ba4cef1335634802f30cc1514d1293, ASSIGN in 333 msec 2024-11-08T03:13:25,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:25,978 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035605978"}]},"ts":"1731035605978"} 2024-11-08T03:13:25,979 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:25,987 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:25,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 801 msec 2024-11-08T03:13:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T03:13:26,193 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-08T03:13:26,193 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:26,194 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:26,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-08T03:13:26,204 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072140 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:26,205 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:26,205 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:26,206 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-08T03:13:26,208 DEBUG [hconnection-0x3fa22986-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:26,209 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36450, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:26,217 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:26,220 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:26,222 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072143 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T03:13:26,222 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T03:13:26,252 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479529 expired clientIPAndPort=172.17.0.2:34838, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:13:27,723 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-08T03:13:27,724 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c503948 to 127.0.0.1:57399 2024-11-08T03:13:27,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:27,724 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:27,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:27,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:27,725 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7c34f73b to 127.0.0.1:57399 2024-11-08T03:13:27,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 230) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=58 (was 36) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2009 (was 2092) 2024-11-08T03:13:27,747 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=58, ProcessCount=11, AvailableMemoryMB=2007 2024-11-08T03:13:27,749 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6fded8d5 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3770b12c 2024-11-08T03:13:27,770 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x6fded8d5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4b30a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:27,771 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x026a91b1 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27b65a51 2024-11-08T03:13:27,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb73602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:27,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-08T03:13:27,789 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:27,789 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:27,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-08T03:13:27,790 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T03:13:27,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741883_1059 (size=419) 2024-11-08T03:13:28,201 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ad7ed95c02821ba2e1ea26601769f945, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:28,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741884_1060 (size=70) 2024-11-08T03:13:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T03:13:28,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:28,255 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479527 expired clientIPAndPort=172.17.0.2:34838, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:13:28,613 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:28,613 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing ad7ed95c02821ba2e1ea26601769f945, disabling compactions & flushes 2024-11-08T03:13:28,613 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,613 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,613 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. after waiting 0 ms 2024-11-08T03:13:28,613 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,613 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,613 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for ad7ed95c02821ba2e1ea26601769f945: 2024-11-08T03:13:28,615 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:28,615 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731035608615"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035608615"}]},"ts":"1731035608615"} 2024-11-08T03:13:28,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:28,618 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:28,618 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035608618"}]},"ts":"1731035608618"} 2024-11-08T03:13:28,620 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:28,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ad7ed95c02821ba2e1ea26601769f945, ASSIGN}] 2024-11-08T03:13:28,638 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ad7ed95c02821ba2e1ea26601769f945, ASSIGN 2024-11-08T03:13:28,638 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ad7ed95c02821ba2e1ea26601769f945, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:28,789 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=ad7ed95c02821ba2e1ea26601769f945, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:28,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure ad7ed95c02821ba2e1ea26601769f945, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T03:13:28,944 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:28,949 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,950 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => ad7ed95c02821ba2e1ea26601769f945, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:28,950 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,950 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:28,950 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,950 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,953 INFO [StoreOpener-ad7ed95c02821ba2e1ea26601769f945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,956 INFO [StoreOpener-ad7ed95c02821ba2e1ea26601769f945-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad7ed95c02821ba2e1ea26601769f945 columnFamilyName testFamily 2024-11-08T03:13:28,956 DEBUG [StoreOpener-ad7ed95c02821ba2e1ea26601769f945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:28,957 INFO [StoreOpener-ad7ed95c02821ba2e1ea26601769f945-1 {}] regionserver.HStore(327): Store=ad7ed95c02821ba2e1ea26601769f945/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:28,958 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,959 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,962 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:13:28,966 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:28,966 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened ad7ed95c02821ba2e1ea26601769f945; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73870885, jitterRate=0.10076196491718292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:28,967 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for ad7ed95c02821ba2e1ea26601769f945: 2024-11-08T03:13:28,969 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945., pid=80, masterSystemTime=1731035608944 2024-11-08T03:13:28,971 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,971 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:13:28,971 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=ad7ed95c02821ba2e1ea26601769f945, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:28,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-08T03:13:28,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure ad7ed95c02821ba2e1ea26601769f945, server=e729c2d9d3a5,45419,1731035408623 in 182 msec 2024-11-08T03:13:28,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T03:13:28,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ad7ed95c02821ba2e1ea26601769f945, ASSIGN in 339 msec 2024-11-08T03:13:28,977 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:28,977 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035608977"}]},"ts":"1731035608977"} 2024-11-08T03:13:28,979 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:28,987 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:28,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.2000 sec 2024-11-08T03:13:30,136 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-08T03:13:30,139 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-08T03:13:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T03:13:30,795 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-08T03:13:30,796 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:30,796 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:30,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-08T03:13:30,804 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072144 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:30,805 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:30,806 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:30,806 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-08T03:13:30,807 DEBUG [hconnection-0x24659e68-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:30,808 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46178, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:30,816 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:30,818 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:30,820 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:13:30,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072147 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:30,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:13:30,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072147 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:30,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T03:13:31,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:33,833 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4017: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4017: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:13:33,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3011 2024-11-08T03:13:33,834 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x026a91b1 to 127.0.0.1:57399 2024-11-08T03:13:33,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:33,834 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:33,834 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:33,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:33,835 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6fded8d5 to 127.0.0.1:57399 2024-11-08T03:13:33,847 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=53 (was 58), ProcessCount=11 (was 11), AvailableMemoryMB=1905 (was 2007) 2024-11-08T03:13:33,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=53, ProcessCount=11, AvailableMemoryMB=1905 2024-11-08T03:13:33,859 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x010b041f to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1597e5a0 2024-11-08T03:13:33,884 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x010b041f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecec146, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:33,884 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x785308bb to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6aabaf72 2024-11-08T03:13:33,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335965c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:33,898 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:33,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T03:13:33,899 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:33,900 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:33,900 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-08T03:13:33,900 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T03:13:33,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741885_1061 (size=421) 2024-11-08T03:13:34,308 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f8b0bbc081a42d35d0ee9ef0f78f5273, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:34,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741886_1062 (size=72) 2024-11-08T03:13:34,323 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:13:34,716 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:34,717 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f8b0bbc081a42d35d0ee9ef0f78f5273, disabling compactions & flushes 2024-11-08T03:13:34,717 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:34,717 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:34,717 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. after waiting 0 ms 2024-11-08T03:13:34,717 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:34,717 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:34,717 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f8b0bbc081a42d35d0ee9ef0f78f5273: 2024-11-08T03:13:34,718 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:34,718 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731035614718"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035614718"}]},"ts":"1731035614718"} 2024-11-08T03:13:34,721 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:34,722 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:34,722 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035614722"}]},"ts":"1731035614722"} 2024-11-08T03:13:34,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:34,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=f8b0bbc081a42d35d0ee9ef0f78f5273, ASSIGN}] 2024-11-08T03:13:34,776 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=f8b0bbc081a42d35d0ee9ef0f78f5273, ASSIGN 2024-11-08T03:13:34,776 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=f8b0bbc081a42d35d0ee9ef0f78f5273, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T03:13:34,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=f8b0bbc081a42d35d0ee9ef0f78f5273, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:34,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure f8b0bbc081a42d35d0ee9ef0f78f5273, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:35,082 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:35,087 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:35,087 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => f8b0bbc081a42d35d0ee9ef0f78f5273, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:35,088 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,088 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:35,088 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,088 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,090 INFO [StoreOpener-f8b0bbc081a42d35d0ee9ef0f78f5273-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,092 INFO [StoreOpener-f8b0bbc081a42d35d0ee9ef0f78f5273-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8b0bbc081a42d35d0ee9ef0f78f5273 columnFamilyName testFamily 2024-11-08T03:13:35,092 DEBUG [StoreOpener-f8b0bbc081a42d35d0ee9ef0f78f5273-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:35,093 INFO [StoreOpener-f8b0bbc081a42d35d0ee9ef0f78f5273-1 {}] regionserver.HStore(327): Store=f8b0bbc081a42d35d0ee9ef0f78f5273/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:35,094 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,095 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,097 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:13:35,100 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:35,101 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened f8b0bbc081a42d35d0ee9ef0f78f5273; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72560348, jitterRate=0.0812334418296814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:35,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for f8b0bbc081a42d35d0ee9ef0f78f5273: 2024-11-08T03:13:35,103 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273., pid=83, masterSystemTime=1731035615082 2024-11-08T03:13:35,105 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:35,105 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:13:35,105 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=f8b0bbc081a42d35d0ee9ef0f78f5273, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:35,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T03:13:35,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure f8b0bbc081a42d35d0ee9ef0f78f5273, server=e729c2d9d3a5,45419,1731035408623 in 178 msec 2024-11-08T03:13:35,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T03:13:35,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=f8b0bbc081a42d35d0ee9ef0f78f5273, ASSIGN in 335 msec 2024-11-08T03:13:35,111 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:35,112 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035615112"}]},"ts":"1731035615112"} 2024-11-08T03:13:35,113 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:35,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:35,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.2280 sec 2024-11-08T03:13:36,142 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-08T03:13:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T03:13:36,904 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-08T03:13:36,904 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:36,904 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:36,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-08T03:13:36,915 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072148 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:36,917 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:36,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:36,917 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T03:13:36,919 DEBUG [hconnection-0x6629e6c4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:36,920 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46192, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:36,928 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:36,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:36,939 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:13:36,943 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:13:36,943 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T03:13:38,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T03:13:38,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:38,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T03:13:38,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:40,483 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479513 expired clientIPAndPort=172.17.0.2:46760, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:13:40,591 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T03:13:43,965 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:13:43,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:46192 deadline: 1731035625942, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T03:13:43,966 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T03:13:43,966 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 18166936420483072151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T03:13:43,967 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7029 2024-11-08T03:13:43,967 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x785308bb to 127.0.0.1:57399 2024-11-08T03:13:43,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:43,967 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:43,968 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:43,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:43,968 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x010b041f to 127.0.0.1:57399 2024-11-08T03:13:43,976 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=74 (was 53) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1711 (was 1905) 2024-11-08T03:13:43,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=74, ProcessCount=11, AvailableMemoryMB=1711 2024-11-08T03:13:43,987 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0677467d to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@59e144eb 2024-11-08T03:13:43,995 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0677467d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e22752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:43,996 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27d32501 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@188d79c 2024-11-08T03:13:44,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df02d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:44,010 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:44,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T03:13:44,012 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:44,012 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:44,012 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-08T03:13:44,013 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T03:13:44,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741887_1063 (size=427) 2024-11-08T03:13:44,422 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f2191e8e536892ecf43f8304a12a0f1b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:44,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741888_1064 (size=78) 2024-11-08T03:13:44,829 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:44,829 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f2191e8e536892ecf43f8304a12a0f1b, disabling compactions & flushes 2024-11-08T03:13:44,829 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:44,829 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:44,829 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. after waiting 0 ms 2024-11-08T03:13:44,829 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:44,829 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:44,829 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f2191e8e536892ecf43f8304a12a0f1b: 2024-11-08T03:13:44,830 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:44,830 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731035624830"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035624830"}]},"ts":"1731035624830"} 2024-11-08T03:13:44,832 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:44,833 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:44,833 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035624833"}]},"ts":"1731035624833"} 2024-11-08T03:13:44,834 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:44,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=f2191e8e536892ecf43f8304a12a0f1b, ASSIGN}] 2024-11-08T03:13:44,883 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=f2191e8e536892ecf43f8304a12a0f1b, ASSIGN 2024-11-08T03:13:44,884 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=f2191e8e536892ecf43f8304a12a0f1b, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T03:13:45,035 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=f2191e8e536892ecf43f8304a12a0f1b, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:45,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure f2191e8e536892ecf43f8304a12a0f1b, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:45,189 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:45,193 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:45,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => f2191e8e536892ecf43f8304a12a0f1b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:45,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:45,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,196 INFO [StoreOpener-f2191e8e536892ecf43f8304a12a0f1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,198 INFO [StoreOpener-f2191e8e536892ecf43f8304a12a0f1b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2191e8e536892ecf43f8304a12a0f1b columnFamilyName testFamily 2024-11-08T03:13:45,198 DEBUG [StoreOpener-f2191e8e536892ecf43f8304a12a0f1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:45,199 INFO [StoreOpener-f2191e8e536892ecf43f8304a12a0f1b-1 {}] regionserver.HStore(327): Store=f2191e8e536892ecf43f8304a12a0f1b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:45,200 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,200 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,203 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:13:45,207 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:45,208 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened f2191e8e536892ecf43f8304a12a0f1b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60819709, jitterRate=-0.09371571242809296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:45,209 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for f2191e8e536892ecf43f8304a12a0f1b: 2024-11-08T03:13:45,209 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., pid=86, masterSystemTime=1731035625188 2024-11-08T03:13:45,211 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:45,211 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:13:45,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=f2191e8e536892ecf43f8304a12a0f1b, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:45,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T03:13:45,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure f2191e8e536892ecf43f8304a12a0f1b, server=e729c2d9d3a5,45419,1731035408623 in 178 msec 2024-11-08T03:13:45,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T03:13:45,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=f2191e8e536892ecf43f8304a12a0f1b, ASSIGN in 334 msec 2024-11-08T03:13:45,220 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:45,220 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035625220"}]},"ts":"1731035625220"} 2024-11-08T03:13:45,222 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:45,235 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:45,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.2260 sec 2024-11-08T03:13:46,779 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:36007,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:13:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T03:13:47,016 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-08T03:13:47,016 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:47,016 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:47,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-08T03:13:47,020 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072153 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:47,022 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:47,022 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:47,022 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T03:13:47,024 DEBUG [hconnection-0x6ad15f20-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:47,025 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57082, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:47,034 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:47,036 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:13:47,037 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:13:47,038 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0677467d {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:13:47,039 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x0677467d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:47,040 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57094, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:13:47,043 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:13:47,491 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479511 expired clientIPAndPort=172.17.0.2:46760, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:13:48,144 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T03:13:48,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T03:13:48,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:50,695 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:13:53,052 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-08T03:13:53,052 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-08T03:13:53,053 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:13:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T03:13:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:13:54,066 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T03:13:55,195 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:13:55,602 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479506 expired clientIPAndPort=172.17.0.2:50302, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:13:56,042 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1971ms 2024-11-08T03:13:56,042 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1971ms 2024-11-08T03:13:56,042 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:13:56,043 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T03:13:53.053Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-08T03:13:56.042Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1971ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T03:13:56,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9007 2024-11-08T03:13:56,044 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x27d32501 to 127.0.0.1:57399 2024-11-08T03:13:56,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:56,044 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:13:56,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:13:56,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:13:56,045 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0677467d to 127.0.0.1:57399 2024-11-08T03:13:56,058 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=225 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=171 (was 74) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1431 (was 1711) 2024-11-08T03:13:56,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=225, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=171, ProcessCount=11, AvailableMemoryMB=1430 2024-11-08T03:13:56,068 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d3d762f to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f7207f6 2024-11-08T03:13:56,141 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x6d3d762f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@186cdec2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:56,144 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2fc4ac0d to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@789d9a0f 2024-11-08T03:13:56,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e97173d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:13:56,159 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:13:56,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-08T03:13:56,161 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:13:56,161 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:56,161 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-08T03:13:56,161 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:13:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T03:13:56,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741889_1065 (size=415) 2024-11-08T03:13:56,572 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f82d269bbcf4e0d7b25f66076193879c, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:13:56,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741890_1066 (size=66) 2024-11-08T03:13:56,984 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:56,984 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f82d269bbcf4e0d7b25f66076193879c, disabling compactions & flushes 2024-11-08T03:13:56,984 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:56,984 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:56,984 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. after waiting 0 ms 2024-11-08T03:13:56,985 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:56,985 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:56,985 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f82d269bbcf4e0d7b25f66076193879c: 2024-11-08T03:13:56,986 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:13:56,987 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731035636986"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035636986"}]},"ts":"1731035636986"} 2024-11-08T03:13:56,990 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:13:56,991 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:13:56,991 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035636991"}]},"ts":"1731035636991"} 2024-11-08T03:13:56,994 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:13:57,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f82d269bbcf4e0d7b25f66076193879c, ASSIGN}] 2024-11-08T03:13:57,041 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f82d269bbcf4e0d7b25f66076193879c, ASSIGN 2024-11-08T03:13:57,042 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f82d269bbcf4e0d7b25f66076193879c, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T03:13:57,193 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=f82d269bbcf4e0d7b25f66076193879c, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:57,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure f82d269bbcf4e0d7b25f66076193879c, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:13:57,348 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:57,358 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:57,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => f82d269bbcf4e0d7b25f66076193879c, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:13:57,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:13:57,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,361 INFO [StoreOpener-f82d269bbcf4e0d7b25f66076193879c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,363 INFO [StoreOpener-f82d269bbcf4e0d7b25f66076193879c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f82d269bbcf4e0d7b25f66076193879c columnFamilyName testFamily 2024-11-08T03:13:57,363 DEBUG [StoreOpener-f82d269bbcf4e0d7b25f66076193879c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:13:57,364 INFO [StoreOpener-f82d269bbcf4e0d7b25f66076193879c-1 {}] regionserver.HStore(327): Store=f82d269bbcf4e0d7b25f66076193879c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:13:57,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,368 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:13:57,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:13:57,371 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened f82d269bbcf4e0d7b25f66076193879c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66630680, jitterRate=-0.007125496864318848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:13:57,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for f82d269bbcf4e0d7b25f66076193879c: 2024-11-08T03:13:57,372 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c., pid=89, masterSystemTime=1731035637348 2024-11-08T03:13:57,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:57,374 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:13:57,374 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=f82d269bbcf4e0d7b25f66076193879c, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:13:57,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T03:13:57,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure f82d269bbcf4e0d7b25f66076193879c, server=e729c2d9d3a5,45419,1731035408623 in 181 msec 2024-11-08T03:13:57,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-08T03:13:57,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f82d269bbcf4e0d7b25f66076193879c, ASSIGN in 337 msec 2024-11-08T03:13:57,379 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:13:57,380 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035637379"}]},"ts":"1731035637379"} 2024-11-08T03:13:57,381 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:13:57,391 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:13:57,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.2320 sec 2024-11-08T03:13:58,205 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479499 expired clientIPAndPort=172.17.0.2:38116, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:13:58,205 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479500 expired clientIPAndPort=172.17.0.2:38116, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:13:58,205 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479502 expired clientIPAndPort=172.17.0.2:38116, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:13:58,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T03:13:58,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T03:13:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T03:13:59,165 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-08T03:13:59,165 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:13:59,166 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:59,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-08T03:13:59,172 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072160 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:59,174 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T03:13:59,174 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:13:59,174 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-08T03:13:59,178 DEBUG [hconnection-0x779467fc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:13:59,179 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47966, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:13:59,187 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:13:59,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T03:13:59,190 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T03:13:59,190 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072163 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:59,191 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T03:13:59,191 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072163 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:13:59,191 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T03:14:00,147 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-08T03:14:00,567 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:14:02,896 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:14:03,410 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479493 expired clientIPAndPort=172.17.0.2:38122, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:14:06,939 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T03:14:07,916 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479488 expired clientIPAndPort=172.17.0.2:38126, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:08,193 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10002: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9001ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10002: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9001ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9001ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9001ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T03:14:08,194 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9003 2024-11-08T03:14:08,195 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2fc4ac0d to 127.0.0.1:57399 2024-11-08T03:14:08,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:08,195 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:14:08,195 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:14:08,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:08,196 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d3d762f to 127.0.0.1:57399 2024-11-08T03:14:08,209 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=160 (was 171), ProcessCount=11 (was 11), AvailableMemoryMB=1262 (was 1430) 2024-11-08T03:14:08,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=160, ProcessCount=11, AvailableMemoryMB=1262 2024-11-08T03:14:08,222 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74ed87fe to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@756f7267 2024-11-08T03:14:08,250 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x74ed87fe {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3723d143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:14:08,251 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x608016b3 to 127.0.0.1:57399 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@555d9000 2024-11-08T03:14:08,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d172f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T03:14:08,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T03:14:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T03:14:08,270 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T03:14:08,270 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:14:08,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-08T03:14:08,272 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T03:14:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T03:14:08,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741891_1067 (size=431) 2024-11-08T03:14:08,687 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1333a0e940001fc8ee6f2add343267b9, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e 2024-11-08T03:14:08,692 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:14:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741892_1068 (size=82) 2024-11-08T03:14:09,099 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:14:09,099 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1333a0e940001fc8ee6f2add343267b9, disabling compactions & flushes 2024-11-08T03:14:09,099 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,099 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,099 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. after waiting 0 ms 2024-11-08T03:14:09,099 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,099 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,099 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1333a0e940001fc8ee6f2add343267b9: 2024-11-08T03:14:09,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T03:14:09,100 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731035649100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731035649100"}]},"ts":"1731035649100"} 2024-11-08T03:14:09,102 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T03:14:09,103 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T03:14:09,103 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035649103"}]},"ts":"1731035649103"} 2024-11-08T03:14:09,105 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T03:14:09,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1333a0e940001fc8ee6f2add343267b9, ASSIGN}] 2024-11-08T03:14:09,158 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1333a0e940001fc8ee6f2add343267b9, ASSIGN 2024-11-08T03:14:09,159 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1333a0e940001fc8ee6f2add343267b9, ASSIGN; state=OFFLINE, location=e729c2d9d3a5,45419,1731035408623; forceNewPlan=false, retain=false 2024-11-08T03:14:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T03:14:09,310 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=1333a0e940001fc8ee6f2add343267b9, regionState=OPENING, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:14:09,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 1333a0e940001fc8ee6f2add343267b9, server=e729c2d9d3a5,45419,1731035408623}] 2024-11-08T03:14:09,468 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e729c2d9d3a5,45419,1731035408623 2024-11-08T03:14:09,478 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,479 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 1333a0e940001fc8ee6f2add343267b9, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9.', STARTKEY => '', ENDKEY => ''} 2024-11-08T03:14:09,479 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,479 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T03:14:09,480 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,480 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,483 INFO [StoreOpener-1333a0e940001fc8ee6f2add343267b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,485 INFO [StoreOpener-1333a0e940001fc8ee6f2add343267b9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1333a0e940001fc8ee6f2add343267b9 columnFamilyName testFamily 2024-11-08T03:14:09,486 DEBUG [StoreOpener-1333a0e940001fc8ee6f2add343267b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T03:14:09,487 INFO [StoreOpener-1333a0e940001fc8ee6f2add343267b9-1 {}] regionserver.HStore(327): Store=1333a0e940001fc8ee6f2add343267b9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T03:14:09,488 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,489 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,491 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:09,494 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T03:14:09,495 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 1333a0e940001fc8ee6f2add343267b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75381036, jitterRate=0.12326496839523315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T03:14:09,495 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 1333a0e940001fc8ee6f2add343267b9: 2024-11-08T03:14:09,496 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., pid=92, masterSystemTime=1731035649468 2024-11-08T03:14:09,498 DEBUG [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,498 INFO [RS_OPEN_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:09,498 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=1333a0e940001fc8ee6f2add343267b9, regionState=OPEN, openSeqNum=2, regionLocation=e729c2d9d3a5,45419,1731035408623 2024-11-08T03:14:09,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-08T03:14:09,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 1333a0e940001fc8ee6f2add343267b9, server=e729c2d9d3a5,45419,1731035408623 in 187 msec 2024-11-08T03:14:09,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T03:14:09,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1333a0e940001fc8ee6f2add343267b9, ASSIGN in 345 msec 2024-11-08T03:14:09,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T03:14:09,505 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731035649505"}]},"ts":"1731035649505"} 2024-11-08T03:14:09,506 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T03:14:09,550 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T03:14:09,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.2840 sec 2024-11-08T03:14:10,019 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479486 expired clientIPAndPort=172.17.0.2:38126, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T03:14:11,276 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-08T03:14:11,276 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T03:14:11,277 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:14:11,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45419 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-08T03:14:11,283 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 18166936420483072164 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T03:14:11,286 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T03:14:11,286 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T03:14:11,286 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T03:14:11,289 DEBUG [hconnection-0x1413dd1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:14:11,290 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57342, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:14:11,297 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T03:14:11,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T03:14:11,299 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T03:14:11,300 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x74ed87fe {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e729c2d9d3a5,45419,1731035408623, seqNum=-1] 2024-11-08T03:14:11,300 DEBUG [ReadOnlyZKClient-127.0.0.1:57399@0x74ed87fe {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T03:14:11,301 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57356, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T03:14:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:14:11,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:14:12,150 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T03:14:12,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-08T03:14:12,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-08T03:14:12,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:14:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T03:14:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2] 2024-11-08T03:14:13,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45419 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T03:14:14,302 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms 2024-11-08T03:14:14,302 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms 2024-11-08T03:14:14,302 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9., hostname=e729c2d9d3a5,45419,1731035408623, seqNum=2 from cache 2024-11-08T03:14:14,303 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T03:14:12.312Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-08T03:14:14.302Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e729c2d9d3a5:45419 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T03:14:14,305 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2024-11-08T03:14:14,305 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x608016b3 to 127.0.0.1:57399 2024-11-08T03:14:14,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:14,305 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T03:14:14,305 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T03:14:14,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:14,305 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74ed87fe to 127.0.0.1:57399 2024-11-08T03:14:14,317 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=231 (was 227) - Thread LEAK? -, OpenFileDescriptor=449 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=147 (was 160), ProcessCount=11 (was 11), AvailableMemoryMB=1260 (was 1262) 2024-11-08T03:14:14,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-08T03:14:14,318 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T03:14:14,318 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x574a453f to 127.0.0.1:57399 2024-11-08T03:14:14,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:14,318 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T03:14:14,318 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1145010739, stopped=false 2024-11-08T03:14:14,318 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=e729c2d9d3a5,39565,1731035407811 2024-11-08T03:14:14,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T03:14:14,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T03:14:14,357 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-08T03:14:14,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:14:14,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:14:14,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:14,358 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T03:14:14,358 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T03:14:14,358 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'e729c2d9d3a5,45419,1731035408623' ***** 2024-11-08T03:14:14,358 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-08T03:14:14,359 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T03:14:14,359 INFO [RS:0;e729c2d9d3a5:45419 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T03:14:14,359 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-08T03:14:14,359 INFO [RS:0;e729c2d9d3a5:45419 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T03:14:14,360 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 1333a0e940001fc8ee6f2add343267b9 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 0b82769fbf29b51a6e6507fe4c11db8d 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 577523dc5c55dcc13eabe84cb05dd3b2 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for f2191e8e536892ecf43f8304a12a0f1b 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 3d77e8660e3bb21e5a6ae8c9b8811473 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 4b725c0119f334af117396c8418b7162 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 80ca2120f66e2dbe59de9827fecc552f 2024-11-08T03:14:14,361 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 496e8ff6915da176f69774373f81da22 2024-11-08T03:14:14,361 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1333a0e940001fc8ee6f2add343267b9, disabling compactions & flushes 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 981c15823b40f09b5aafa63cd023e8c0 2024-11-08T03:14:14,362 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:14,362 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for e4dc35a373b38f6d59efad851dae653f 2024-11-08T03:14:14,362 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. after waiting 0 ms 2024-11-08T03:14:14,362 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 006d5905ba2c972f64fe852b7b6a56e7 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 9eefc29c5ad773a11362212e16e34d43 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 64a03bd514341f2ca7c009db1aa26e9d 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for c74b6a41d3522743e243c16780bdd0c7 2024-11-08T03:14:14,362 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 8874f3ae66c0b430c92fcb070abb9ee1 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for ad7ed95c02821ba2e1ea26601769f945 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for af931b4fe418ddc9248d7e94b4f6fa28 2024-11-08T03:14:14,363 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1333a0e940001fc8ee6f2add343267b9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 8a0070350f48c6b23867fc41ac42767b 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 19566d9f13bc4850b9dbe183d03f6f25 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 565426ad1c8cfb593f23db513d128881 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 53a46b3b2a374085c1bbdb5620bdf1e5 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for f82d269bbcf4e0d7b25f66076193879c 2024-11-08T03:14:14,363 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 5d61dadc50789866d6f56149ebb69543 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 7bb69b6e2882ff2ac3bf9bfd0167d883 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 01e439eb5eab7a7ae47039d307f0cfd0 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for f8b0bbc081a42d35d0ee9ef0f78f5273 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for d6de0e77e271698fa27c712288e77738 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for c3ba4cef1335634802f30cc1514d1293 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1224): stopping server e729c2d9d3a5,45419,1731035408623 2024-11-08T03:14:14,364 DEBUG [RS:0;e729c2d9d3a5:45419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T03:14:14,364 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T03:14:14,365 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-08T03:14:14,365 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-08T03:14:14,365 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T03:14:14,365 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T03:14:14,365 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T03:14:14,365 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T03:14:14,365 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T03:14:14,365 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-08T03:14:14,365 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 0b82769fbf29b51a6e6507fe4c11db8d, 1333a0e940001fc8ee6f2add343267b9, 1588230740, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:14,413 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/.tmp/testFamily/d38f41241e8a4f51bb67d2e10f8a1d04 is 61, key is row-0/testFamily:testQualifier/1731035651292/Put/seqid=0 2024-11-08T03:14:14,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741893_1069 (size=5049) 2024-11-08T03:14:14,430 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/.tmp/info/683606d3d0ed45a3a1b2dd946682eccc is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7./info:regioninfo/1731035444987/Put/seqid=0 2024-11-08T03:14:14,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741894_1070 (size=51873) 2024-11-08T03:14:14,566 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 0b82769fbf29b51a6e6507fe4c11db8d, 1333a0e940001fc8ee6f2add343267b9, 1588230740, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:14,766 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 0b82769fbf29b51a6e6507fe4c11db8d, 1333a0e940001fc8ee6f2add343267b9, 1588230740, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:14,825 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/.tmp/testFamily/d38f41241e8a4f51bb67d2e10f8a1d04 2024-11-08T03:14:14,825 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T03:14:14,836 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/.tmp/info/683606d3d0ed45a3a1b2dd946682eccc 2024-11-08T03:14:14,872 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/.tmp/testFamily/d38f41241e8a4f51bb67d2e10f8a1d04 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/testFamily/d38f41241e8a4f51bb67d2e10f8a1d04 2024-11-08T03:14:14,882 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/testFamily/d38f41241e8a4f51bb67d2e10f8a1d04, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:14,887 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1333a0e940001fc8ee6f2add343267b9 in 524ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:14,890 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/.tmp/table/319f8bf33f5a40bd8e41825d60a054dc is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731035444999/Put/seqid=0 2024-11-08T03:14:14,893 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1333a0e940001fc8ee6f2add343267b9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:14,896 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:14,896 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1333a0e940001fc8ee6f2add343267b9: 2024-11-08T03:14:14,896 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731035648266.1333a0e940001fc8ee6f2add343267b9. 2024-11-08T03:14:14,896 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0b82769fbf29b51a6e6507fe4c11db8d, disabling compactions & flushes 2024-11-08T03:14:14,896 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:14:14,896 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:14:14,896 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. after waiting 0 ms 2024-11-08T03:14:14,896 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:14:14,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741895_1071 (size=9821) 2024-11-08T03:14:14,896 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0b82769fbf29b51a6e6507fe4c11db8d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:14,898 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/.tmp/table/319f8bf33f5a40bd8e41825d60a054dc 2024-11-08T03:14:14,906 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/.tmp/info/683606d3d0ed45a3a1b2dd946682eccc as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/info/683606d3d0ed45a3a1b2dd946682eccc 2024-11-08T03:14:14,913 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/info/683606d3d0ed45a3a1b2dd946682eccc, entries=290, sequenceid=149, filesize=50.7 K 2024-11-08T03:14:14,914 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/.tmp/table/319f8bf33f5a40bd8e41825d60a054dc as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/table/319f8bf33f5a40bd8e41825d60a054dc 2024-11-08T03:14:14,916 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/.tmp/testFamily/827b1c24636f4b6caa0dfa119a757175 is 61, key is row-0/testFamily:testQualifier/1731035578159/Put/seqid=0 2024-11-08T03:14:14,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741896_1072 (size=5049) 2024-11-08T03:14:14,922 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/table/319f8bf33f5a40bd8e41825d60a054dc, entries=58, sequenceid=149, filesize=9.6 K 2024-11-08T03:14:14,923 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 558ms, sequenceid=149, compaction requested=false 2024-11-08T03:14:14,927 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-08T03:14:14,928 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T03:14:14,928 INFO [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T03:14:14,928 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T03:14:14,928 DEBUG [RS_CLOSE_META-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T03:14:14,967 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 0b82769fbf29b51a6e6507fe4c11db8d, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:15,018 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:14:15,087 INFO [regionserver/e729c2d9d3a5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T03:14:15,087 INFO [regionserver/e729c2d9d3a5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T03:14:15,167 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 0b82769fbf29b51a6e6507fe4c11db8d, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:15,321 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/.tmp/testFamily/827b1c24636f4b6caa0dfa119a757175 2024-11-08T03:14:15,331 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/.tmp/testFamily/827b1c24636f4b6caa0dfa119a757175 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/testFamily/827b1c24636f4b6caa0dfa119a757175 2024-11-08T03:14:15,339 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/testFamily/827b1c24636f4b6caa0dfa119a757175, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:15,340 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0b82769fbf29b51a6e6507fe4c11db8d in 444ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:15,345 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_1_-true/0b82769fbf29b51a6e6507fe4c11db8d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:15,346 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:14:15,346 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0b82769fbf29b51a6e6507fe4c11db8d: 2024-11-08T03:14:15,346 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731035575123.0b82769fbf29b51a6e6507fe4c11db8d. 2024-11-08T03:14:15,346 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 577523dc5c55dcc13eabe84cb05dd3b2, disabling compactions & flushes 2024-11-08T03:14:15,346 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:14:15,346 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:14:15,346 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. after waiting 0 ms 2024-11-08T03:14:15,346 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:14:15,346 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 577523dc5c55dcc13eabe84cb05dd3b2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:15,364 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/.tmp/testFamily/36d53610f5f64fe2851c68bbc2459f31 is 61, key is row-0/testFamily:testQualifier/1731035544654/Put/seqid=0 2024-11-08T03:14:15,367 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-08T03:14:15,368 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:15,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741897_1073 (size=5049) 2024-11-08T03:14:15,568 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:15,768 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 577523dc5c55dcc13eabe84cb05dd3b2, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f2191e8e536892ecf43f8304a12a0f1b, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:15,770 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/.tmp/testFamily/36d53610f5f64fe2851c68bbc2459f31 2024-11-08T03:14:15,779 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/.tmp/testFamily/36d53610f5f64fe2851c68bbc2459f31 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/testFamily/36d53610f5f64fe2851c68bbc2459f31 2024-11-08T03:14:15,788 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/testFamily/36d53610f5f64fe2851c68bbc2459f31, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:15,789 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 577523dc5c55dcc13eabe84cb05dd3b2 in 443ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:15,793 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/577523dc5c55dcc13eabe84cb05dd3b2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:15,794 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:14:15,794 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 577523dc5c55dcc13eabe84cb05dd3b2: 2024-11-08T03:14:15,794 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731035541623.577523dc5c55dcc13eabe84cb05dd3b2. 2024-11-08T03:14:15,794 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f2191e8e536892ecf43f8304a12a0f1b, disabling compactions & flushes 2024-11-08T03:14:15,794 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:14:15,794 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:14:15,794 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. after waiting 0 ms 2024-11-08T03:14:15,794 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:14:15,794 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f2191e8e536892ecf43f8304a12a0f1b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:15,817 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/.tmp/testFamily/1ccfdea927bd40a99178300ed354550b is 61, key is row-0/testFamily:testQualifier/1731035627027/Put/seqid=0 2024-11-08T03:14:15,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741898_1074 (size=5049) 2024-11-08T03:14:15,852 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/.tmp/testFamily/1ccfdea927bd40a99178300ed354550b 2024-11-08T03:14:15,863 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/.tmp/testFamily/1ccfdea927bd40a99178300ed354550b as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/testFamily/1ccfdea927bd40a99178300ed354550b 2024-11-08T03:14:15,876 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/testFamily/1ccfdea927bd40a99178300ed354550b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:15,877 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f2191e8e536892ecf43f8304a12a0f1b in 83ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:15,884 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/f2191e8e536892ecf43f8304a12a0f1b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:15,885 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:14:15,885 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f2191e8e536892ecf43f8304a12a0f1b: 2024-11-08T03:14:15,885 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731035624010.f2191e8e536892ecf43f8304a12a0f1b. 2024-11-08T03:14:15,886 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3d77e8660e3bb21e5a6ae8c9b8811473, disabling compactions & flushes 2024-11-08T03:14:15,886 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:14:15,886 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:14:15,886 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. after waiting 0 ms 2024-11-08T03:14:15,886 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:14:15,886 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3d77e8660e3bb21e5a6ae8c9b8811473 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:15,911 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/.tmp/testFamily/5b97e3362b36419ab77abd2fd8f45d63 is 61, key is row-0/testFamily:testQualifier/1731035494771/Put/seqid=0 2024-11-08T03:14:15,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741899_1075 (size=5049) 2024-11-08T03:14:15,969 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:16,026 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -279807653226479481 expired clientIPAndPort=172.17.0.2:45892, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:14:16,169 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 3d77e8660e3bb21e5a6ae8c9b8811473, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:16,317 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/.tmp/testFamily/5b97e3362b36419ab77abd2fd8f45d63 2024-11-08T03:14:16,327 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/.tmp/testFamily/5b97e3362b36419ab77abd2fd8f45d63 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/testFamily/5b97e3362b36419ab77abd2fd8f45d63 2024-11-08T03:14:16,337 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/testFamily/5b97e3362b36419ab77abd2fd8f45d63, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:16,338 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3d77e8660e3bb21e5a6ae8c9b8811473 in 452ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:16,343 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_0_-false/3d77e8660e3bb21e5a6ae8c9b8811473/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:16,344 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:14:16,344 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3d77e8660e3bb21e5a6ae8c9b8811473: 2024-11-08T03:14:16,344 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731035491741.3d77e8660e3bb21e5a6ae8c9b8811473. 2024-11-08T03:14:16,344 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4b725c0119f334af117396c8418b7162, disabling compactions & flushes 2024-11-08T03:14:16,344 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:14:16,344 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:14:16,344 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. after waiting 0 ms 2024-11-08T03:14:16,345 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:14:16,345 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4b725c0119f334af117396c8418b7162 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:16,367 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/.tmp/testFamily/8195bb41371442e2ba11349b744bc034 is 61, key is row-0/testFamily:testQualifier/1731035535475/Put/seqid=0 2024-11-08T03:14:16,369 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 24 regions to close 2024-11-08T03:14:16,369 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 496e8ff6915da176f69774373f81da22, 4b725c0119f334af117396c8418b7162, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:16,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741900_1076 (size=5049) 2024-11-08T03:14:16,373 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/.tmp/testFamily/8195bb41371442e2ba11349b744bc034 2024-11-08T03:14:16,380 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/.tmp/testFamily/8195bb41371442e2ba11349b744bc034 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/testFamily/8195bb41371442e2ba11349b744bc034 2024-11-08T03:14:16,387 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/testFamily/8195bb41371442e2ba11349b744bc034, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:16,387 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4b725c0119f334af117396c8418b7162 in 42ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:16,391 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_0_-false/4b725c0119f334af117396c8418b7162/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:16,392 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:14:16,392 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4b725c0119f334af117396c8418b7162: 2024-11-08T03:14:16,392 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731035532448.4b725c0119f334af117396c8418b7162. 2024-11-08T03:14:16,392 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 80ca2120f66e2dbe59de9827fecc552f, disabling compactions & flushes 2024-11-08T03:14:16,392 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:14:16,392 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:14:16,392 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. after waiting 0 ms 2024-11-08T03:14:16,392 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:14:16,392 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 80ca2120f66e2dbe59de9827fecc552f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:16,406 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/.tmp/testFamily/2efa598adc4f402ab8ce992b72f60a4c is 61, key is row-0/testFamily:testQualifier/1731035592510/Put/seqid=0 2024-11-08T03:14:16,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741901_1077 (size=5049) 2024-11-08T03:14:16,570 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 496e8ff6915da176f69774373f81da22, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:16,770 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 496e8ff6915da176f69774373f81da22, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 80ca2120f66e2dbe59de9827fecc552f, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:16,811 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/.tmp/testFamily/2efa598adc4f402ab8ce992b72f60a4c 2024-11-08T03:14:16,818 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/.tmp/testFamily/2efa598adc4f402ab8ce992b72f60a4c as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/testFamily/2efa598adc4f402ab8ce992b72f60a4c 2024-11-08T03:14:16,823 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/testFamily/2efa598adc4f402ab8ce992b72f60a4c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:16,824 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 80ca2120f66e2dbe59de9827fecc552f in 432ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:16,828 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_1_-true/80ca2120f66e2dbe59de9827fecc552f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:16,829 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:14:16,829 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 80ca2120f66e2dbe59de9827fecc552f: 2024-11-08T03:14:16,829 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731035589490.80ca2120f66e2dbe59de9827fecc552f. 2024-11-08T03:14:16,829 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 496e8ff6915da176f69774373f81da22, disabling compactions & flushes 2024-11-08T03:14:16,829 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:14:16,829 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:14:16,829 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. after waiting 0 ms 2024-11-08T03:14:16,829 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:14:16,829 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 496e8ff6915da176f69774373f81da22 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-08T03:14:16,845 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/.tmp/info/cfcb7dd6a2a547148a7f3939be4d94a6 is 45, key is default/info:d/1731035443426/Put/seqid=0 2024-11-08T03:14:16,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741902_1078 (size=5037) 2024-11-08T03:14:16,970 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 496e8ff6915da176f69774373f81da22, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:17,171 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 496e8ff6915da176f69774373f81da22, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:17,249 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/.tmp/info/cfcb7dd6a2a547148a7f3939be4d94a6 2024-11-08T03:14:17,257 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/.tmp/info/cfcb7dd6a2a547148a7f3939be4d94a6 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/info/cfcb7dd6a2a547148a7f3939be4d94a6 2024-11-08T03:14:17,263 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/info/cfcb7dd6a2a547148a7f3939be4d94a6, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T03:14:17,264 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 496e8ff6915da176f69774373f81da22 in 435ms, sequenceid=6, compaction requested=false 2024-11-08T03:14:17,268 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/namespace/496e8ff6915da176f69774373f81da22/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T03:14:17,269 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:14:17,269 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 496e8ff6915da176f69774373f81da22: 2024-11-08T03:14:17,269 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731035441963.496e8ff6915da176f69774373f81da22. 2024-11-08T03:14:17,269 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 981c15823b40f09b5aafa63cd023e8c0, disabling compactions & flushes 2024-11-08T03:14:17,269 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:14:17,269 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:14:17,269 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. after waiting 0 ms 2024-11-08T03:14:17,269 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:14:17,269 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 981c15823b40f09b5aafa63cd023e8c0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:17,283 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/.tmp/testFamily/c0a2ffe271f84dcdab0a24eadc9a0d0a is 61, key is row-0/testFamily:testQualifier/1731035513181/Put/seqid=0 2024-11-08T03:14:17,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741903_1079 (size=5049) 2024-11-08T03:14:17,371 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-08T03:14:17,371 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:17,572 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 981c15823b40f09b5aafa63cd023e8c0, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:17,690 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/.tmp/testFamily/c0a2ffe271f84dcdab0a24eadc9a0d0a 2024-11-08T03:14:17,703 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/.tmp/testFamily/c0a2ffe271f84dcdab0a24eadc9a0d0a as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/testFamily/c0a2ffe271f84dcdab0a24eadc9a0d0a 2024-11-08T03:14:17,709 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/testFamily/c0a2ffe271f84dcdab0a24eadc9a0d0a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:17,710 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 981c15823b40f09b5aafa63cd023e8c0 in 441ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:17,715 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/981c15823b40f09b5aafa63cd023e8c0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:17,715 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:14:17,715 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 981c15823b40f09b5aafa63cd023e8c0: 2024-11-08T03:14:17,715 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731035510150.981c15823b40f09b5aafa63cd023e8c0. 2024-11-08T03:14:17,715 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e4dc35a373b38f6d59efad851dae653f, disabling compactions & flushes 2024-11-08T03:14:17,715 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:14:17,716 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:14:17,716 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. after waiting 0 ms 2024-11-08T03:14:17,716 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:14:17,716 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e4dc35a373b38f6d59efad851dae653f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:17,732 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/.tmp/testFamily/b519bc10cc184d34bf159cb338d3252c is 61, key is row-0/testFamily:testQualifier/1731035488644/Put/seqid=0 2024-11-08T03:14:17,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741904_1080 (size=5049) 2024-11-08T03:14:17,772 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:17,972 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, e4dc35a373b38f6d59efad851dae653f, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:18,137 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/.tmp/testFamily/b519bc10cc184d34bf159cb338d3252c 2024-11-08T03:14:18,151 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/.tmp/testFamily/b519bc10cc184d34bf159cb338d3252c as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/testFamily/b519bc10cc184d34bf159cb338d3252c 2024-11-08T03:14:18,157 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/testFamily/b519bc10cc184d34bf159cb338d3252c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:18,157 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e4dc35a373b38f6d59efad851dae653f in 441ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:18,161 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNextAsync_0_-false/e4dc35a373b38f6d59efad851dae653f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:18,161 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:14:18,161 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e4dc35a373b38f6d59efad851dae653f: 2024-11-08T03:14:18,162 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731035485616.e4dc35a373b38f6d59efad851dae653f. 2024-11-08T03:14:18,162 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 006d5905ba2c972f64fe852b7b6a56e7, disabling compactions & flushes 2024-11-08T03:14:18,162 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:14:18,162 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:14:18,162 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. after waiting 0 ms 2024-11-08T03:14:18,162 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:14:18,162 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 006d5905ba2c972f64fe852b7b6a56e7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:18,173 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:18,179 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/.tmp/testFamily/ca65390d2b5c417b89fcf98582cd7d8b is 61, key is row-0/testFamily:testQualifier/1731035446806/Put/seqid=0 2024-11-08T03:14:18,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741905_1081 (size=5049) 2024-11-08T03:14:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T03:14:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T03:14:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T03:14:18,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T03:14:18,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T03:14:18,373 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 19 regions to close 2024-11-08T03:14:18,373 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:18,573 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 006d5905ba2c972f64fe852b7b6a56e7, 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:18,583 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/.tmp/testFamily/ca65390d2b5c417b89fcf98582cd7d8b 2024-11-08T03:14:18,594 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/.tmp/testFamily/ca65390d2b5c417b89fcf98582cd7d8b as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/testFamily/ca65390d2b5c417b89fcf98582cd7d8b 2024-11-08T03:14:18,601 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/testFamily/ca65390d2b5c417b89fcf98582cd7d8b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:18,602 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 006d5905ba2c972f64fe852b7b6a56e7 in 440ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:18,607 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/006d5905ba2c972f64fe852b7b6a56e7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:18,608 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:14:18,608 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 006d5905ba2c972f64fe852b7b6a56e7: 2024-11-08T03:14:18,608 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731035443754.006d5905ba2c972f64fe852b7b6a56e7. 2024-11-08T03:14:18,608 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9eefc29c5ad773a11362212e16e34d43, disabling compactions & flushes 2024-11-08T03:14:18,609 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:14:18,609 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:14:18,609 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. after waiting 0 ms 2024-11-08T03:14:18,609 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:14:18,609 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9eefc29c5ad773a11362212e16e34d43 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:18,628 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/.tmp/testFamily/31ac5b23c9c542f9b0146509a0c1bc1a is 61, key is row-0/testFamily:testQualifier/1731035462113/Put/seqid=0 2024-11-08T03:14:18,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741906_1082 (size=5049) 2024-11-08T03:14:18,774 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:18,974 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, 9eefc29c5ad773a11362212e16e34d43, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:19,034 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/.tmp/testFamily/31ac5b23c9c542f9b0146509a0c1bc1a 2024-11-08T03:14:19,048 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/.tmp/testFamily/31ac5b23c9c542f9b0146509a0c1bc1a as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/testFamily/31ac5b23c9c542f9b0146509a0c1bc1a 2024-11-08T03:14:19,056 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/testFamily/31ac5b23c9c542f9b0146509a0c1bc1a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:19,057 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9eefc29c5ad773a11362212e16e34d43 in 448ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:19,061 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/9eefc29c5ad773a11362212e16e34d43/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:19,062 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:14:19,062 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9eefc29c5ad773a11362212e16e34d43: 2024-11-08T03:14:19,062 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731035459078.9eefc29c5ad773a11362212e16e34d43. 2024-11-08T03:14:19,062 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 64a03bd514341f2ca7c009db1aa26e9d, disabling compactions & flushes 2024-11-08T03:14:19,062 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:14:19,062 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:14:19,062 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. after waiting 0 ms 2024-11-08T03:14:19,062 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:14:19,062 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 64a03bd514341f2ca7c009db1aa26e9d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:19,077 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/.tmp/testFamily/6355f9a3d2d2463e96c690ae35429ce0 is 61, key is row-0/testFamily:testQualifier/1731035482534/Put/seqid=0 2024-11-08T03:14:19,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741907_1083 (size=5049) 2024-11-08T03:14:19,174 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:19,374 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 17 regions to close 2024-11-08T03:14:19,375 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 64a03bd514341f2ca7c009db1aa26e9d, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:19,483 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/.tmp/testFamily/6355f9a3d2d2463e96c690ae35429ce0 2024-11-08T03:14:19,498 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/.tmp/testFamily/6355f9a3d2d2463e96c690ae35429ce0 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/testFamily/6355f9a3d2d2463e96c690ae35429ce0 2024-11-08T03:14:19,505 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/testFamily/6355f9a3d2d2463e96c690ae35429ce0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:19,506 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 64a03bd514341f2ca7c009db1aa26e9d in 444ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:19,511 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/64a03bd514341f2ca7c009db1aa26e9d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:19,511 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:14:19,511 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 64a03bd514341f2ca7c009db1aa26e9d: 2024-11-08T03:14:19,512 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731035479507.64a03bd514341f2ca7c009db1aa26e9d. 2024-11-08T03:14:19,512 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c74b6a41d3522743e243c16780bdd0c7, disabling compactions & flushes 2024-11-08T03:14:19,512 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:14:19,512 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:14:19,512 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. after waiting 0 ms 2024-11-08T03:14:19,512 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:14:19,512 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c74b6a41d3522743e243c16780bdd0c7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:19,528 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/.tmp/testFamily/6c7cf089baa54ee1a4d84d33c3b9c3ca is 61, key is row-0/testFamily:testQualifier/1731035509016/Put/seqid=0 2024-11-08T03:14:19,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741908_1084 (size=5049) 2024-11-08T03:14:19,575 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:19,775 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, c74b6a41d3522743e243c16780bdd0c7, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:19,933 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/.tmp/testFamily/6c7cf089baa54ee1a4d84d33c3b9c3ca 2024-11-08T03:14:19,945 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/.tmp/testFamily/6c7cf089baa54ee1a4d84d33c3b9c3ca as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/testFamily/6c7cf089baa54ee1a4d84d33c3b9c3ca 2024-11-08T03:14:19,952 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/testFamily/6c7cf089baa54ee1a4d84d33c3b9c3ca, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:19,953 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c74b6a41d3522743e243c16780bdd0c7 in 441ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:19,957 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_0_-false/c74b6a41d3522743e243c16780bdd0c7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:19,958 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:14:19,958 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c74b6a41d3522743e243c16780bdd0c7: 2024-11-08T03:14:19,958 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731035505982.c74b6a41d3522743e243c16780bdd0c7. 2024-11-08T03:14:19,958 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8874f3ae66c0b430c92fcb070abb9ee1, disabling compactions & flushes 2024-11-08T03:14:19,958 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:14:19,958 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:14:19,958 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. after waiting 0 ms 2024-11-08T03:14:19,958 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:14:19,958 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8874f3ae66c0b430c92fcb070abb9ee1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:19,973 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/.tmp/testFamily/39e9b979cdd44846bc70f808b32dd462 is 61, key is row-0/testFamily:testQualifier/1731035523323/Put/seqid=0 2024-11-08T03:14:19,976 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:19,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741909_1085 (size=5049) 2024-11-08T03:14:20,176 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:20,376 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 15 regions to close 2024-11-08T03:14:20,377 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8874f3ae66c0b430c92fcb070abb9ee1, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:20,379 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/.tmp/testFamily/39e9b979cdd44846bc70f808b32dd462 2024-11-08T03:14:20,394 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/.tmp/testFamily/39e9b979cdd44846bc70f808b32dd462 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/testFamily/39e9b979cdd44846bc70f808b32dd462 2024-11-08T03:14:20,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:14:20,400 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/testFamily/39e9b979cdd44846bc70f808b32dd462, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:20,401 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8874f3ae66c0b430c92fcb070abb9ee1 in 443ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:20,406 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8874f3ae66c0b430c92fcb070abb9ee1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:20,407 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:14:20,407 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8874f3ae66c0b430c92fcb070abb9ee1: 2024-11-08T03:14:20,407 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731035520299.8874f3ae66c0b430c92fcb070abb9ee1. 2024-11-08T03:14:20,407 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ad7ed95c02821ba2e1ea26601769f945, disabling compactions & flushes 2024-11-08T03:14:20,407 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:14:20,407 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:14:20,407 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. after waiting 0 ms 2024-11-08T03:14:20,407 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:14:20,407 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ad7ed95c02821ba2e1ea26601769f945 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:20,423 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/.tmp/testFamily/a4f76ef4e2314d1cb9a186a18bfe8db9 is 61, key is row-0/testFamily:testQualifier/1731035610810/Put/seqid=0 2024-11-08T03:14:20,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741910_1086 (size=5049) 2024-11-08T03:14:20,577 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:20,777 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8a0070350f48c6b23867fc41ac42767b, ad7ed95c02821ba2e1ea26601769f945, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:20,828 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/.tmp/testFamily/a4f76ef4e2314d1cb9a186a18bfe8db9 2024-11-08T03:14:20,869 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/.tmp/testFamily/a4f76ef4e2314d1cb9a186a18bfe8db9 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/testFamily/a4f76ef4e2314d1cb9a186a18bfe8db9 2024-11-08T03:14:20,875 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/testFamily/a4f76ef4e2314d1cb9a186a18bfe8db9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:20,876 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ad7ed95c02821ba2e1ea26601769f945 in 469ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnNext_1_-true/ad7ed95c02821ba2e1ea26601769f945/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:20,880 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ad7ed95c02821ba2e1ea26601769f945: 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731035607786.ad7ed95c02821ba2e1ea26601769f945. 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing af931b4fe418ddc9248d7e94b4f6fa28, disabling compactions & flushes 2024-11-08T03:14:20,880 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. after waiting 0 ms 2024-11-08T03:14:20,880 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:14:20,881 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing af931b4fe418ddc9248d7e94b4f6fa28 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:20,895 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/.tmp/testFamily/0ff08382c2a94f24a46f055d6cc0f858 is 61, key is row-0/testFamily:testQualifier/1731035474296/Put/seqid=0 2024-11-08T03:14:20,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741911_1087 (size=5049) 2024-11-08T03:14:20,978 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8a0070350f48c6b23867fc41ac42767b, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:21,178 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8a0070350f48c6b23867fc41ac42767b, af931b4fe418ddc9248d7e94b4f6fa28, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:21,301 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/.tmp/testFamily/0ff08382c2a94f24a46f055d6cc0f858 2024-11-08T03:14:21,315 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/.tmp/testFamily/0ff08382c2a94f24a46f055d6cc0f858 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/testFamily/0ff08382c2a94f24a46f055d6cc0f858 2024-11-08T03:14:21,320 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/testFamily/0ff08382c2a94f24a46f055d6cc0f858, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:21,321 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for af931b4fe418ddc9248d7e94b4f6fa28 in 441ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:21,325 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextException_0_-false/af931b4fe418ddc9248d7e94b4f6fa28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:21,326 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:14:21,326 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for af931b4fe418ddc9248d7e94b4f6fa28: 2024-11-08T03:14:21,326 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731035471264.af931b4fe418ddc9248d7e94b4f6fa28. 2024-11-08T03:14:21,326 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8a0070350f48c6b23867fc41ac42767b, disabling compactions & flushes 2024-11-08T03:14:21,326 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:14:21,326 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:14:21,326 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. after waiting 0 ms 2024-11-08T03:14:21,326 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:14:21,326 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8a0070350f48c6b23867fc41ac42767b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:21,342 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/.tmp/testFamily/2bef9e6f06594d7987fc8dfb192a0520 is 61, key is row-0/testFamily:testQualifier/1731035477420/Put/seqid=0 2024-11-08T03:14:21,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741912_1088 (size=5049) 2024-11-08T03:14:21,378 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 12 regions to close 2024-11-08T03:14:21,378 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8a0070350f48c6b23867fc41ac42767b, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:21,579 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, 8a0070350f48c6b23867fc41ac42767b, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:21,748 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/.tmp/testFamily/2bef9e6f06594d7987fc8dfb192a0520 2024-11-08T03:14:21,762 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/.tmp/testFamily/2bef9e6f06594d7987fc8dfb192a0520 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/testFamily/2bef9e6f06594d7987fc8dfb192a0520 2024-11-08T03:14:21,767 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/testFamily/2bef9e6f06594d7987fc8dfb192a0520, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:21,768 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8a0070350f48c6b23867fc41ac42767b in 442ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:21,772 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_0_-false/8a0070350f48c6b23867fc41ac42767b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:21,772 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:14:21,772 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8a0070350f48c6b23867fc41ac42767b: 2024-11-08T03:14:21,772 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731035474389.8a0070350f48c6b23867fc41ac42767b. 2024-11-08T03:14:21,773 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 19566d9f13bc4850b9dbe183d03f6f25, disabling compactions & flushes 2024-11-08T03:14:21,773 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:14:21,773 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:14:21,773 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. after waiting 0 ms 2024-11-08T03:14:21,773 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:14:21,773 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 19566d9f13bc4850b9dbe183d03f6f25 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:21,779 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:21,786 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/.tmp/testFamily/e926b6b3b9824522aae6def3749d111e is 61, key is row-0/testFamily:testQualifier/1731035581289/Put/seqid=0 2024-11-08T03:14:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741913_1089 (size=5049) 2024-11-08T03:14:21,980 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:22,180 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 19566d9f13bc4850b9dbe183d03f6f25, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:22,192 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/.tmp/testFamily/e926b6b3b9824522aae6def3749d111e 2024-11-08T03:14:22,208 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/.tmp/testFamily/e926b6b3b9824522aae6def3749d111e as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/testFamily/e926b6b3b9824522aae6def3749d111e 2024-11-08T03:14:22,215 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/testFamily/e926b6b3b9824522aae6def3749d111e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:22,216 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 19566d9f13bc4850b9dbe183d03f6f25 in 443ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:22,220 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnRenewLease_1_-true/19566d9f13bc4850b9dbe183d03f6f25/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:22,220 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:14:22,220 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 19566d9f13bc4850b9dbe183d03f6f25: 2024-11-08T03:14:22,220 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731035578257.19566d9f13bc4850b9dbe183d03f6f25. 2024-11-08T03:14:22,221 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 565426ad1c8cfb593f23db513d128881, disabling compactions & flushes 2024-11-08T03:14:22,221 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:14:22,221 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:14:22,221 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. after waiting 0 ms 2024-11-08T03:14:22,221 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:14:22,221 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 565426ad1c8cfb593f23db513d128881 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:22,240 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/.tmp/testFamily/f8838aa3fa2041649667281df0406ec9 is 61, key is row-0/testFamily:testQualifier/1731035598613/Put/seqid=0 2024-11-08T03:14:22,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741914_1090 (size=5049) 2024-11-08T03:14:22,380 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 10 regions to close 2024-11-08T03:14:22,380 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:22,581 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 53a46b3b2a374085c1bbdb5620bdf1e5, 565426ad1c8cfb593f23db513d128881, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:22,646 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/.tmp/testFamily/f8838aa3fa2041649667281df0406ec9 2024-11-08T03:14:22,660 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/.tmp/testFamily/f8838aa3fa2041649667281df0406ec9 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/testFamily/f8838aa3fa2041649667281df0406ec9 2024-11-08T03:14:22,668 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/testFamily/f8838aa3fa2041649667281df0406ec9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:22,669 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 565426ad1c8cfb593f23db513d128881 in 448ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:22,674 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnClose_1_-true/565426ad1c8cfb593f23db513d128881/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:22,675 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:14:22,675 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 565426ad1c8cfb593f23db513d128881: 2024-11-08T03:14:22,675 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731035595588.565426ad1c8cfb593f23db513d128881. 2024-11-08T03:14:22,675 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 53a46b3b2a374085c1bbdb5620bdf1e5, disabling compactions & flushes 2024-11-08T03:14:22,675 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:14:22,675 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:14:22,676 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. after waiting 0 ms 2024-11-08T03:14:22,676 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:14:22,676 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 53a46b3b2a374085c1bbdb5620bdf1e5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:22,693 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/.tmp/testFamily/02654b2c38b74b39bbf6d7b57ee9b58b is 61, key is row-0/testFamily:testQualifier/1731035450000/Put/seqid=0 2024-11-08T03:14:22,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741915_1091 (size=5049) 2024-11-08T03:14:22,781 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 53a46b3b2a374085c1bbdb5620bdf1e5, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:22,981 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 53a46b3b2a374085c1bbdb5620bdf1e5, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:23,098 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/.tmp/testFamily/02654b2c38b74b39bbf6d7b57ee9b58b 2024-11-08T03:14:23,105 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/.tmp/testFamily/02654b2c38b74b39bbf6d7b57ee9b58b as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/testFamily/02654b2c38b74b39bbf6d7b57ee9b58b 2024-11-08T03:14:23,111 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/testFamily/02654b2c38b74b39bbf6d7b57ee9b58b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:23,111 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 53a46b3b2a374085c1bbdb5620bdf1e5 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:23,115 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/53a46b3b2a374085c1bbdb5620bdf1e5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:23,116 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:14:23,116 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 53a46b3b2a374085c1bbdb5620bdf1e5: 2024-11-08T03:14:23,116 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731035446965.53a46b3b2a374085c1bbdb5620bdf1e5. 2024-11-08T03:14:23,116 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f82d269bbcf4e0d7b25f66076193879c, disabling compactions & flushes 2024-11-08T03:14:23,116 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:14:23,116 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:14:23,116 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. after waiting 0 ms 2024-11-08T03:14:23,116 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:14:23,116 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f82d269bbcf4e0d7b25f66076193879c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:23,130 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/.tmp/testFamily/65cba6d98ca940ab997973b7ee93cd7c is 61, key is row-0/testFamily:testQualifier/1731035639182/Put/seqid=0 2024-11-08T03:14:23,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741916_1092 (size=5049) 2024-11-08T03:14:23,182 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:23,382 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 8 regions to close 2024-11-08T03:14:23,382 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1603): Online Regions={f82d269bbcf4e0d7b25f66076193879c=hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c., 5d61dadc50789866d6f56149ebb69543=testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543., 7bb69b6e2882ff2ac3bf9bfd0167d883=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883., 01e439eb5eab7a7ae47039d307f0cfd0=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., f8b0bbc081a42d35d0ee9ef0f78f5273=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273., d6de0e77e271698fa27c712288e77738=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738., c3ba4cef1335634802f30cc1514d1293=testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293., ff1ef2a898f94dec827a806db3b081f4=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.} 2024-11-08T03:14:23,382 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f82d269bbcf4e0d7b25f66076193879c, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:23,537 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/.tmp/testFamily/65cba6d98ca940ab997973b7ee93cd7c 2024-11-08T03:14:23,550 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/.tmp/testFamily/65cba6d98ca940ab997973b7ee93cd7c as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/testFamily/65cba6d98ca940ab997973b7ee93cd7c 2024-11-08T03:14:23,556 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/testFamily/65cba6d98ca940ab997973b7ee93cd7c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:23,557 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f82d269bbcf4e0d7b25f66076193879c in 441ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:23,561 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNext_1_-true/f82d269bbcf4e0d7b25f66076193879c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:23,561 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:14:23,562 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f82d269bbcf4e0d7b25f66076193879c: 2024-11-08T03:14:23,562 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731035636159.f82d269bbcf4e0d7b25f66076193879c. 2024-11-08T03:14:23,562 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5d61dadc50789866d6f56149ebb69543, disabling compactions & flushes 2024-11-08T03:14:23,562 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:14:23,562 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:14:23,562 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. after waiting 0 ms 2024-11-08T03:14:23,562 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:14:23,562 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 5d61dadc50789866d6f56149ebb69543 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:23,576 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/.tmp/testFamily/2164fc74e8694c3cb5baadd8eb23ee4b is 61, key is row-0/testFamily:testQualifier/1731035504359/Put/seqid=0 2024-11-08T03:14:23,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741917_1093 (size=5049) 2024-11-08T03:14:23,582 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:23,783 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:23,981 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/.tmp/testFamily/2164fc74e8694c3cb5baadd8eb23ee4b 2024-11-08T03:14:23,983 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 5d61dadc50789866d6f56149ebb69543, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:23,989 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/.tmp/testFamily/2164fc74e8694c3cb5baadd8eb23ee4b as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/testFamily/2164fc74e8694c3cb5baadd8eb23ee4b 2024-11-08T03:14:23,996 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/testFamily/2164fc74e8694c3cb5baadd8eb23ee4b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:23,997 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5d61dadc50789866d6f56149ebb69543 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:24,002 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_0_-false/5d61dadc50789866d6f56149ebb69543/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:24,003 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:14:24,003 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5d61dadc50789866d6f56149ebb69543: 2024-11-08T03:14:24,003 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731035501332.5d61dadc50789866d6f56149ebb69543. 2024-11-08T03:14:24,003 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7bb69b6e2882ff2ac3bf9bfd0167d883, disabling compactions & flushes 2024-11-08T03:14:24,003 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:14:24,003 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:14:24,003 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. after waiting 0 ms 2024-11-08T03:14:24,003 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:14:24,004 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7bb69b6e2882ff2ac3bf9bfd0167d883 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:24,019 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/.tmp/testFamily/d965f51d96574759a30582aec5f4ca0d is 61, key is row-0/testFamily:testQualifier/1731035550820/Put/seqid=0 2024-11-08T03:14:24,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741918_1094 (size=5049) 2024-11-08T03:14:24,183 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:24,384 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 6 regions to close 2024-11-08T03:14:24,384 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1603): Online Regions={7bb69b6e2882ff2ac3bf9bfd0167d883=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883., 01e439eb5eab7a7ae47039d307f0cfd0=testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0., f8b0bbc081a42d35d0ee9ef0f78f5273=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273., d6de0e77e271698fa27c712288e77738=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738., c3ba4cef1335634802f30cc1514d1293=testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293., ff1ef2a898f94dec827a806db3b081f4=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.} 2024-11-08T03:14:24,384 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, 7bb69b6e2882ff2ac3bf9bfd0167d883, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:24,425 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/.tmp/testFamily/d965f51d96574759a30582aec5f4ca0d 2024-11-08T03:14:24,439 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/.tmp/testFamily/d965f51d96574759a30582aec5f4ca0d as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/testFamily/d965f51d96574759a30582aec5f4ca0d 2024-11-08T03:14:24,447 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/testFamily/d965f51d96574759a30582aec5f4ca0d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:24,448 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7bb69b6e2882ff2ac3bf9bfd0167d883 in 444ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:24,452 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7bb69b6e2882ff2ac3bf9bfd0167d883/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:24,453 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:14:24,453 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7bb69b6e2882ff2ac3bf9bfd0167d883: 2024-11-08T03:14:24,453 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731035547790.7bb69b6e2882ff2ac3bf9bfd0167d883. 2024-11-08T03:14:24,453 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 01e439eb5eab7a7ae47039d307f0cfd0, disabling compactions & flushes 2024-11-08T03:14:24,453 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:24,453 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:24,453 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. after waiting 0 ms 2024-11-08T03:14:24,453 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:24,453 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 01e439eb5eab7a7ae47039d307f0cfd0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:24,467 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/.tmp/testFamily/9292318e1b3d4973bc2655bf036df390 is 61, key is row-0/testFamily:testQualifier/1731035586405/Put/seqid=0 2024-11-08T03:14:24,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741919_1095 (size=5049) 2024-11-08T03:14:24,584 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:24,785 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on 01e439eb5eab7a7ae47039d307f0cfd0, c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:24,872 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/.tmp/testFamily/9292318e1b3d4973bc2655bf036df390 2024-11-08T03:14:24,878 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/.tmp/testFamily/9292318e1b3d4973bc2655bf036df390 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/testFamily/9292318e1b3d4973bc2655bf036df390 2024-11-08T03:14:24,884 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/testFamily/9292318e1b3d4973bc2655bf036df390, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:24,885 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 01e439eb5eab7a7ae47039d307f0cfd0 in 432ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:24,889 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/01e439eb5eab7a7ae47039d307f0cfd0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:24,889 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:24,889 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 01e439eb5eab7a7ae47039d307f0cfd0: 2024-11-08T03:14:24,889 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731035583374.01e439eb5eab7a7ae47039d307f0cfd0. 2024-11-08T03:14:24,889 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f8b0bbc081a42d35d0ee9ef0f78f5273, disabling compactions & flushes 2024-11-08T03:14:24,889 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:14:24,890 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:14:24,890 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. after waiting 0 ms 2024-11-08T03:14:24,890 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:14:24,890 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f8b0bbc081a42d35d0ee9ef0f78f5273 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:24,903 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/.tmp/testFamily/1d195bdde0374f879665d032b4858c1d is 61, key is row-0/testFamily:testQualifier/1731035616922/Put/seqid=0 2024-11-08T03:14:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741920_1096 (size=5049) 2024-11-08T03:14:24,985 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:25,186 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, f8b0bbc081a42d35d0ee9ef0f78f5273, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:25,308 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/.tmp/testFamily/1d195bdde0374f879665d032b4858c1d 2024-11-08T03:14:25,321 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/.tmp/testFamily/1d195bdde0374f879665d032b4858c1d as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/testFamily/1d195bdde0374f879665d032b4858c1d 2024-11-08T03:14:25,326 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/testFamily/1d195bdde0374f879665d032b4858c1d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:25,327 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f8b0bbc081a42d35d0ee9ef0f78f5273 in 437ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:25,331 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/f8b0bbc081a42d35d0ee9ef0f78f5273/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:25,332 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:14:25,332 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f8b0bbc081a42d35d0ee9ef0f78f5273: 2024-11-08T03:14:25,332 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731035613898.f8b0bbc081a42d35d0ee9ef0f78f5273. 2024-11-08T03:14:25,332 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d6de0e77e271698fa27c712288e77738, disabling compactions & flushes 2024-11-08T03:14:25,332 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:14:25,332 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:14:25,332 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. after waiting 0 ms 2024-11-08T03:14:25,333 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:14:25,333 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d6de0e77e271698fa27c712288e77738 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:25,348 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/.tmp/testFamily/b2367775de7a484dba141f8275dced37 is 61, key is row-0/testFamily:testQualifier/1731035566027/Put/seqid=0 2024-11-08T03:14:25,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741921_1097 (size=5049) 2024-11-08T03:14:25,386 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-11-08T03:14:25,386 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1603): Online Regions={d6de0e77e271698fa27c712288e77738=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738., c3ba4cef1335634802f30cc1514d1293=testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293., ff1ef2a898f94dec827a806db3b081f4=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.} 2024-11-08T03:14:25,386 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:25,461 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T03:14:25,586 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, d6de0e77e271698fa27c712288e77738, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:25,753 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/.tmp/testFamily/b2367775de7a484dba141f8275dced37 2024-11-08T03:14:25,758 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/.tmp/testFamily/b2367775de7a484dba141f8275dced37 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/testFamily/b2367775de7a484dba141f8275dced37 2024-11-08T03:14:25,764 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/testFamily/b2367775de7a484dba141f8275dced37, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:25,765 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d6de0e77e271698fa27c712288e77738 in 432ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:25,768 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d6de0e77e271698fa27c712288e77738/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:25,769 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:14:25,769 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d6de0e77e271698fa27c712288e77738: 2024-11-08T03:14:25,769 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731035562997.d6de0e77e271698fa27c712288e77738. 2024-11-08T03:14:25,769 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c3ba4cef1335634802f30cc1514d1293, disabling compactions & flushes 2024-11-08T03:14:25,769 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:14:25,769 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:14:25,769 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. after waiting 0 ms 2024-11-08T03:14:25,769 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:14:25,769 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c3ba4cef1335634802f30cc1514d1293 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:25,783 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/.tmp/testFamily/0ddda30be7a544f7906eb5570beca5bb is 61, key is row-0/testFamily:testQualifier/1731035606212/Put/seqid=0 2024-11-08T03:14:25,787 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:25,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741922_1098 (size=5049) 2024-11-08T03:14:25,987 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:26,168 ERROR [LeaseRenewer:jenkins@localhost:36007 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:36007,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T03:14:26,187 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on c3ba4cef1335634802f30cc1514d1293, ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:26,189 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/.tmp/testFamily/0ddda30be7a544f7906eb5570beca5bb 2024-11-08T03:14:26,195 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/.tmp/testFamily/0ddda30be7a544f7906eb5570beca5bb as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/testFamily/0ddda30be7a544f7906eb5570beca5bb 2024-11-08T03:14:26,199 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/testFamily/0ddda30be7a544f7906eb5570beca5bb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:26,200 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c3ba4cef1335634802f30cc1514d1293 in 431ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:26,204 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/default/testNormalScanTimeoutOnClose_1_-true/c3ba4cef1335634802f30cc1514d1293/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:26,204 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:14:26,204 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c3ba4cef1335634802f30cc1514d1293: 2024-11-08T03:14:26,204 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731035605186.c3ba4cef1335634802f30cc1514d1293. 2024-11-08T03:14:26,204 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ff1ef2a898f94dec827a806db3b081f4, disabling compactions & flushes 2024-11-08T03:14:26,204 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:14:26,205 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:14:26,205 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. after waiting 0 ms 2024-11-08T03:14:26,205 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:14:26,205 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ff1ef2a898f94dec827a806db3b081f4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T03:14:26,218 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/.tmp/testFamily/3396ce2a211a47b3a1fa9580f3e0af4f is 61, key is row-0/testFamily:testQualifier/1731035553934/Put/seqid=0 2024-11-08T03:14:26,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741923_1099 (size=5049) 2024-11-08T03:14:26,387 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1599): Waiting on 1 regions to close 2024-11-08T03:14:26,388 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1603): Online Regions={ff1ef2a898f94dec827a806db3b081f4=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4.} 2024-11-08T03:14:26,388 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:26,588 DEBUG [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1629): Waiting on ff1ef2a898f94dec827a806db3b081f4 2024-11-08T03:14:26,622 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/.tmp/testFamily/3396ce2a211a47b3a1fa9580f3e0af4f 2024-11-08T03:14:26,628 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/.tmp/testFamily/3396ce2a211a47b3a1fa9580f3e0af4f as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/testFamily/3396ce2a211a47b3a1fa9580f3e0af4f 2024-11-08T03:14:26,634 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/testFamily/3396ce2a211a47b3a1fa9580f3e0af4f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T03:14:26,635 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ff1ef2a898f94dec827a806db3b081f4 in 430ms, sequenceid=8, compaction requested=false 2024-11-08T03:14:26,639 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ff1ef2a898f94dec827a806db3b081f4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T03:14:26,639 INFO [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:14:26,639 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ff1ef2a898f94dec827a806db3b081f4: 2024-11-08T03:14:26,640 DEBUG [RS_CLOSE_REGION-regionserver/e729c2d9d3a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731035550907.ff1ef2a898f94dec827a806db3b081f4. 2024-11-08T03:14:26,788 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1250): stopping server e729c2d9d3a5,45419,1731035408623; all regions closed. 2024-11-08T03:14:26,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741834_1010 (size=56731) 2024-11-08T03:14:26,799 DEBUG [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/oldWALs 2024-11-08T03:14:26,799 INFO [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL e729c2d9d3a5%2C45419%2C1731035408623.meta:.meta(num 1731035441670) 2024-11-08T03:14:26,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741833_1009 (size=60562) 2024-11-08T03:14:26,802 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/WALs/e729c2d9d3a5,45419,1731035408623/e729c2d9d3a5%2C45419%2C1731035408623.1731035441146 not finished, retry = 0 2024-11-08T03:14:26,906 DEBUG [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/oldWALs 2024-11-08T03:14:26,906 INFO [RS:0;e729c2d9d3a5:45419 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL e729c2d9d3a5%2C45419%2C1731035408623:(num 1731035441146) 2024-11-08T03:14:26,906 DEBUG [RS:0;e729c2d9d3a5:45419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:26,907 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T03:14:26,907 INFO [RS:0;e729c2d9d3a5:45419 {}] hbase.ChoreService(370): Chore service for: regionserver/e729c2d9d3a5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-11-08T03:14:26,908 INFO [regionserver/e729c2d9d3a5:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T03:14:26,909 INFO [RS:0;e729c2d9d3a5:45419 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45419 2024-11-08T03:14:26,939 INFO [regionserver/e729c2d9d3a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T03:14:26,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T03:14:26,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e729c2d9d3a5,45419,1731035408623 2024-11-08T03:14:26,970 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e729c2d9d3a5,45419,1731035408623] 2024-11-08T03:14:26,970 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing e729c2d9d3a5,45419,1731035408623; numProcessing=1 2024-11-08T03:14:26,986 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/e729c2d9d3a5,45419,1731035408623 already deleted, retry=false 2024-11-08T03:14:26,986 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; e729c2d9d3a5,45419,1731035408623 expired; onlineServers=0 2024-11-08T03:14:26,986 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'e729c2d9d3a5,39565,1731035407811' ***** 2024-11-08T03:14:26,986 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T03:14:26,986 DEBUG [M:0;e729c2d9d3a5:39565 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed7708f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e729c2d9d3a5/172.17.0.2:0 2024-11-08T03:14:26,986 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegionServer(1224): stopping server e729c2d9d3a5,39565,1731035407811 2024-11-08T03:14:26,986 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegionServer(1250): stopping server e729c2d9d3a5,39565,1731035407811; all regions closed. 2024-11-08T03:14:26,986 DEBUG [M:0;e729c2d9d3a5:39565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T03:14:26,987 DEBUG [M:0;e729c2d9d3a5:39565 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T03:14:26,987 DEBUG [M:0;e729c2d9d3a5:39565 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T03:14:26,987 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T03:14:26,987 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster-HFileCleaner.small.0-1731035410995 {}] cleaner.HFileCleaner(306): Exit Thread[master/e729c2d9d3a5:0:becomeActiveMaster-HFileCleaner.small.0-1731035410995,5,FailOnTimeoutGroup] 2024-11-08T03:14:26,987 DEBUG [master/e729c2d9d3a5:0:becomeActiveMaster-HFileCleaner.large.0-1731035410994 {}] cleaner.HFileCleaner(306): Exit Thread[master/e729c2d9d3a5:0:becomeActiveMaster-HFileCleaner.large.0-1731035410994,5,FailOnTimeoutGroup] 2024-11-08T03:14:26,987 INFO [M:0;e729c2d9d3a5:39565 {}] hbase.ChoreService(370): Chore service for: master/e729c2d9d3a5:0 had [] on shutdown 2024-11-08T03:14:26,987 DEBUG [M:0;e729c2d9d3a5:39565 {}] master.HMaster(1733): Stopping service threads 2024-11-08T03:14:26,987 INFO [M:0;e729c2d9d3a5:39565 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T03:14:26,988 INFO [M:0;e729c2d9d3a5:39565 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T03:14:26,988 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T03:14:26,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T03:14:26,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T03:14:26,994 DEBUG [M:0;e729c2d9d3a5:39565 {}] zookeeper.ZKUtil(347): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T03:14:26,995 WARN [M:0;e729c2d9d3a5:39565 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T03:14:26,995 INFO [M:0;e729c2d9d3a5:39565 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-08T03:14:26,995 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T03:14:26,995 INFO [M:0;e729c2d9d3a5:39565 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T03:14:26,995 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T03:14:26,995 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:14:26,995 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:14:26,995 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T03:14:26,995 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:14:26,995 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.63 KB heapSize=520.21 KB 2024-11-08T03:14:27,011 DEBUG [M:0;e729c2d9d3a5:39565 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3939244e04c843298aaa1871279fd69d is 82, key is hbase:meta,,1/info:regioninfo/1731035441809/Put/seqid=0 2024-11-08T03:14:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741924_1100 (size=5672) 2024-11-08T03:14:27,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T03:14:27,070 INFO [RS:0;e729c2d9d3a5:45419 {}] regionserver.HRegionServer(1307): Exiting; stopping=e729c2d9d3a5,45419,1731035408623; zookeeper connection closed. 2024-11-08T03:14:27,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45419-0x10119b5c5050001, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T03:14:27,070 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e1b0f1e {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e1b0f1e 2024-11-08T03:14:27,071 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T03:14:27,416 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3939244e04c843298aaa1871279fd69d 2024-11-08T03:14:27,443 DEBUG [M:0;e729c2d9d3a5:39565 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cca5142b0ce742b181a576075d359e35 is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731035445014/Put/seqid=0 2024-11-08T03:14:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741925_1101 (size=30296) 2024-11-08T03:14:27,850 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=448.08 KB at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cca5142b0ce742b181a576075d359e35 2024-11-08T03:14:27,860 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cca5142b0ce742b181a576075d359e35 2024-11-08T03:14:27,875 DEBUG [M:0;e729c2d9d3a5:39565 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0089d2b5916436abea5f35fd4be11b7 is 69, key is e729c2d9d3a5,45419,1731035408623/rs:state/1731035411011/Put/seqid=0 2024-11-08T03:14:27,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741926_1102 (size=5156) 2024-11-08T03:14:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T03:14:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T03:14:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T03:14:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T03:14:28,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T03:14:28,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T03:14:28,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T03:14:28,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T03:14:28,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T03:14:28,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T03:14:28,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T03:14:28,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T03:14:28,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T03:14:28,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T03:14:28,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T03:14:28,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T03:14:28,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T03:14:28,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T03:14:28,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T03:14:28,281 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0089d2b5916436abea5f35fd4be11b7 2024-11-08T03:14:28,293 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3939244e04c843298aaa1871279fd69d as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3939244e04c843298aaa1871279fd69d 2024-11-08T03:14:28,298 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3939244e04c843298aaa1871279fd69d, entries=8, sequenceid=832, filesize=5.5 K 2024-11-08T03:14:28,299 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cca5142b0ce742b181a576075d359e35 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cca5142b0ce742b181a576075d359e35 2024-11-08T03:14:28,304 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cca5142b0ce742b181a576075d359e35 2024-11-08T03:14:28,304 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cca5142b0ce742b181a576075d359e35, entries=92, sequenceid=832, filesize=29.6 K 2024-11-08T03:14:28,305 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0089d2b5916436abea5f35fd4be11b7 as hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f0089d2b5916436abea5f35fd4be11b7 2024-11-08T03:14:28,310 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36007/user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f0089d2b5916436abea5f35fd4be11b7, entries=1, sequenceid=832, filesize=5.0 K 2024-11-08T03:14:28,311 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.63 KB/459400, heapSize ~519.91 KB/532392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=832, compaction requested=false 2024-11-08T03:14:28,312 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T03:14:28,312 DEBUG [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T03:14:28,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38891 is added to blk_1073741830_1006 (size=514399) 2024-11-08T03:14:28,315 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/8a5853ea-7ba5-0ca6-551c-7c9c7fca794e/MasterData/WALs/e729c2d9d3a5,39565,1731035407811/e729c2d9d3a5%2C39565%2C1731035407811.1731035410313 not finished, retry = 0 2024-11-08T03:14:28,416 INFO [M:0;e729c2d9d3a5:39565 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-08T03:14:28,416 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T03:14:28,416 INFO [M:0;e729c2d9d3a5:39565 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:39565 2024-11-08T03:14:28,456 DEBUG [M:0;e729c2d9d3a5:39565 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/e729c2d9d3a5,39565,1731035407811 already deleted, retry=false 2024-11-08T03:14:28,565 INFO [M:0;e729c2d9d3a5:39565 {}] regionserver.HRegionServer(1307): Exiting; stopping=e729c2d9d3a5,39565,1731035407811; zookeeper connection closed. 2024-11-08T03:14:28,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T03:14:28,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39565-0x10119b5c5050000, quorum=127.0.0.1:57399, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T03:14:28,607 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@183020ad{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T03:14:28,611 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T03:14:28,611 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T03:14:28,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T03:14:28,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/hadoop.log.dir/,STOPPED} 2024-11-08T03:14:28,613 WARN [BP-696983924-172.17.0.2-1731035404563 heartbeating to localhost/127.0.0.1:36007 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T03:14:28,613 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T03:14:28,614 WARN [BP-696983924-172.17.0.2-1731035404563 heartbeating to localhost/127.0.0.1:36007 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-696983924-172.17.0.2-1731035404563 (Datanode Uuid 41a4abef-dbe0-44b3-8368-6caca7824c0d) service to localhost/127.0.0.1:36007 2024-11-08T03:14:28,614 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T03:14:28,615 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/dfs/data/data1/current/BP-696983924-172.17.0.2-1731035404563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T03:14:28,616 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/cluster_da8ddcf0-fc92-488a-3bd7-e8fb0d86d05c/dfs/data/data2/current/BP-696983924-172.17.0.2-1731035404563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T03:14:28,616 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T03:14:28,622 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T03:14:28,622 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T03:14:28,622 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T03:14:28,622 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T03:14:28,623 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/71f3e6f1-6ee5-6646-83c2-820de6425db4/hadoop.log.dir/,STOPPED} 2024-11-08T03:14:28,633 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-08T03:14:28,667 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down