2024-11-08 11:17:04,461 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 11:17:04,478 main DEBUG Took 0.014233 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 11:17:04,479 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 11:17:04,479 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 11:17:04,480 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 11:17:04,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,501 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 11:17:04,516 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,519 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,521 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,522 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,523 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,524 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,525 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,526 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,527 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,528 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,529 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 11:17:04,530 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,531 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 11:17:04,532 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 11:17:04,534 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 11:17:04,536 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 11:17:04,537 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 11:17:04,538 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 11:17:04,539 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 11:17:04,547 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 11:17:04,551 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 11:17:04,553 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 11:17:04,553 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 11:17:04,554 main DEBUG createAppenders(={Console}) 2024-11-08 11:17:04,555 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-08 11:17:04,555 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 11:17:04,555 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-08 11:17:04,556 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 11:17:04,556 main DEBUG OutputStream closed 2024-11-08 11:17:04,557 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 11:17:04,557 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 11:17:04,557 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-08 11:17:04,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 11:17:04,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 11:17:04,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 11:17:04,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 11:17:04,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 11:17:04,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 11:17:04,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 11:17:04,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 11:17:04,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 11:17:04,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 11:17:04,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 11:17:04,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 11:17:04,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 11:17:04,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 11:17:04,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 11:17:04,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 11:17:04,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 11:17:04,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 11:17:04,642 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 11:17:04,642 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-08 11:17:04,642 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 11:17:04,643 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-08T11:17:04,864 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-08 11:17:04,868 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 11:17:04,868 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T11:17:04,943 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86 2024-11-08T11:17:04,976 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T11:17:04,979 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5, deleteOnExit=true 2024-11-08T11:17:04,980 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-08T11:17:04,980 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/test.cache.data in system properties and HBase conf 2024-11-08T11:17:04,981 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T11:17:04,981 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/hadoop.log.dir in system properties and HBase conf 2024-11-08T11:17:04,982 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T11:17:04,982 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T11:17:04,983 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-08T11:17:05,097 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T11:17:05,203 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T11:17:05,206 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T11:17:05,207 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T11:17:05,207 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T11:17:05,208 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T11:17:05,208 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T11:17:05,208 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T11:17:05,209 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T11:17:05,209 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T11:17:05,210 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T11:17:05,210 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/nfs.dump.dir in system properties and HBase conf 2024-11-08T11:17:05,210 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/java.io.tmpdir in system properties and HBase conf 2024-11-08T11:17:05,211 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T11:17:05,211 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T11:17:05,211 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T11:17:06,067 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T11:17:06,148 INFO [Time-limited test {}] log.Log(170): Logging initialized @2598ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T11:17:06,228 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T11:17:06,302 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T11:17:06,330 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T11:17:06,330 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T11:17:06,331 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T11:17:06,344 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T11:17:06,347 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/hadoop.log.dir/,AVAILABLE} 2024-11-08T11:17:06,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T11:17:06,570 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/java.io.tmpdir/jetty-localhost-38065-hadoop-hdfs-3_4_1-tests_jar-_-any-10383874192679678986/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T11:17:06,581 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:38065} 2024-11-08T11:17:06,581 INFO [Time-limited test {}] server.Server(415): Started @3032ms 2024-11-08T11:17:07,029 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T11:17:07,037 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T11:17:07,039 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T11:17:07,039 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T11:17:07,039 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T11:17:07,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/hadoop.log.dir/,AVAILABLE} 2024-11-08T11:17:07,041 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T11:17:07,158 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@183020ad{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/java.io.tmpdir/jetty-localhost-36441-hadoop-hdfs-3_4_1-tests_jar-_-any-11029958188007277809/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T11:17:07,159 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:36441} 2024-11-08T11:17:07,159 INFO [Time-limited test {}] server.Server(415): Started @3611ms 2024-11-08T11:17:07,221 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T11:17:07,634 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/dfs/data/data2/current/BP-503676101-172.17.0.2-1731064625816/current, will proceed with Du for space computation calculation, 2024-11-08T11:17:07,634 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/dfs/data/data1/current/BP-503676101-172.17.0.2-1731064625816/current, will proceed with Du for space computation calculation, 2024-11-08T11:17:07,674 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T11:17:07,730 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x60ab23ba990b34a2 with lease ID 0xf77d8e08f28f6783: Processing first storage report for DS-722f6c85-8fbf-4049-ae27-2d425b50b94f from datanode DatanodeRegistration(127.0.0.1:40011, datanodeUuid=309c3c92-024d-4753-b0a4-40b69f0a5676, infoPort=34689, infoSecurePort=0, ipcPort=39331, storageInfo=lv=-57;cid=testClusterID;nsid=525321484;c=1731064625816) 2024-11-08T11:17:07,731 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60ab23ba990b34a2 with lease ID 0xf77d8e08f28f6783: from storage DS-722f6c85-8fbf-4049-ae27-2d425b50b94f node DatanodeRegistration(127.0.0.1:40011, datanodeUuid=309c3c92-024d-4753-b0a4-40b69f0a5676, infoPort=34689, infoSecurePort=0, ipcPort=39331, storageInfo=lv=-57;cid=testClusterID;nsid=525321484;c=1731064625816), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T11:17:07,731 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x60ab23ba990b34a2 with lease ID 0xf77d8e08f28f6783: Processing first storage report for DS-98388b5b-3293-4cf5-a318-e9c696d93cfd from datanode DatanodeRegistration(127.0.0.1:40011, datanodeUuid=309c3c92-024d-4753-b0a4-40b69f0a5676, infoPort=34689, infoSecurePort=0, ipcPort=39331, storageInfo=lv=-57;cid=testClusterID;nsid=525321484;c=1731064625816) 2024-11-08T11:17:07,731 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60ab23ba990b34a2 with lease ID 0xf77d8e08f28f6783: from storage DS-98388b5b-3293-4cf5-a318-e9c696d93cfd node DatanodeRegistration(127.0.0.1:40011, datanodeUuid=309c3c92-024d-4753-b0a4-40b69f0a5676, infoPort=34689, infoSecurePort=0, ipcPort=39331, storageInfo=lv=-57;cid=testClusterID;nsid=525321484;c=1731064625816), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T11:17:07,775 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86 2024-11-08T11:17:07,851 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/zookeeper_0, clientPort=55518, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T11:17:07,863 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=55518 2024-11-08T11:17:07,878 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:07,882 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:08,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741825_1001 (size=7) 2024-11-08T11:17:08,511 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a with version=8 2024-11-08T11:17:08,511 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/hbase-staging 2024-11-08T11:17:08,635 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T11:17:08,911 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ae52d81aa32e:0 server-side Connection retries=9 2024-11-08T11:17:08,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T11:17:08,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T11:17:08,933 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T11:17:08,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T11:17:08,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T11:17:09,066 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T11:17:09,127 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T11:17:09,136 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T11:17:09,140 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T11:17:09,167 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 11641 (auto-detected) 2024-11-08T11:17:09,169 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T11:17:09,188 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36127 2024-11-08T11:17:09,196 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:09,198 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:09,211 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36127 connecting to ZooKeeper ensemble=127.0.0.1:55518 2024-11-08T11:17:09,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361270x0, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T11:17:09,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36127-0x101879977df0000 connected 2024-11-08T11:17:09,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T11:17:09,282 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T11:17:09,300 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T11:17:09,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36127 2024-11-08T11:17:09,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36127 2024-11-08T11:17:09,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36127 2024-11-08T11:17:09,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36127 2024-11-08T11:17:09,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36127 2024-11-08T11:17:09,332 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a, hbase.cluster.distributed=false 2024-11-08T11:17:09,395 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ae52d81aa32e:0 server-side Connection retries=9 2024-11-08T11:17:09,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T11:17:09,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T11:17:09,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T11:17:09,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T11:17:09,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T11:17:09,398 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T11:17:09,401 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T11:17:09,402 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:33717 2024-11-08T11:17:09,403 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T11:17:09,409 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T11:17:09,410 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:09,414 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:09,417 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:33717 connecting to ZooKeeper ensemble=127.0.0.1:55518 2024-11-08T11:17:09,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337170x0, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T11:17:09,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:337170x0, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T11:17:09,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33717-0x101879977df0001 connected 2024-11-08T11:17:09,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T11:17:09,424 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T11:17:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33717 2024-11-08T11:17:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33717 2024-11-08T11:17:09,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33717 2024-11-08T11:17:09,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33717 2024-11-08T11:17:09,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33717 2024-11-08T11:17:09,429 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:09,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T11:17:09,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T11:17:09,437 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:09,444 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ae52d81aa32e:36127 2024-11-08T11:17:09,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T11:17:09,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T11:17:09,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:09,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:09,459 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T11:17:09,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T11:17:09,460 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ae52d81aa32e,36127,1731064628628 from backup master directory 2024-11-08T11:17:09,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:09,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T11:17:09,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T11:17:09,464 WARN [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T11:17:09,464 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:09,466 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T11:17:09,467 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T11:17:09,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741826_1002 (size=42) 2024-11-08T11:17:09,937 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/hbase.id with ID: c412b867-814f-4e10-ab48-e7bac993cea1 2024-11-08T11:17:09,978 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T11:17:10,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:10,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:10,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741827_1003 (size=196) 2024-11-08T11:17:10,434 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:17:10,436 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T11:17:10,452 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:17:10,457 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T11:17:10,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741828_1004 (size=1189) 2024-11-08T11:17:10,904 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store 2024-11-08T11:17:10,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741829_1005 (size=34) 2024-11-08T11:17:10,924 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T11:17:10,925 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:10,926 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T11:17:10,926 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:17:10,926 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:17:10,926 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T11:17:10,926 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:17:10,927 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:17:10,927 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T11:17:10,929 WARN [master/ae52d81aa32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/.initializing 2024-11-08T11:17:10,929 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/WALs/ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:10,936 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T11:17:10,947 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae52d81aa32e%2C36127%2C1731064628628, suffix=, logDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/WALs/ae52d81aa32e,36127,1731064628628, archiveDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/oldWALs, maxLogs=10 2024-11-08T11:17:10,968 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/WALs/ae52d81aa32e,36127,1731064628628/ae52d81aa32e%2C36127%2C1731064628628.1731064630952, exclude list is [], retry=0 2024-11-08T11:17:10,984 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40011,DS-722f6c85-8fbf-4049-ae27-2d425b50b94f,DISK] 2024-11-08T11:17:10,987 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T11:17:11,022 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/WALs/ae52d81aa32e,36127,1731064628628/ae52d81aa32e%2C36127%2C1731064628628.1731064630952 2024-11-08T11:17:11,022 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34689:34689)] 2024-11-08T11:17:11,023 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:17:11,023 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:11,026 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,027 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T11:17:11,094 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:11,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:11,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T11:17:11,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:11,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:11,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T11:17:11,106 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:11,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:11,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T11:17:11,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:11,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:11,116 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,117 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,128 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T11:17:11,133 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T11:17:11,138 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:17:11,140 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66171401, jitterRate=-0.013969287276268005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T11:17:11,145 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T11:17:11,147 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T11:17:11,176 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f503d72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:11,209 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-08T11:17:11,220 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T11:17:11,220 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T11:17:11,223 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T11:17:11,224 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T11:17:11,229 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-11-08T11:17:11,229 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T11:17:11,253 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T11:17:11,264 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T11:17:11,266 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-08T11:17:11,268 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T11:17:11,269 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T11:17:11,271 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-08T11:17:11,273 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T11:17:11,275 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T11:17:11,277 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-08T11:17:11,278 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T11:17:11,279 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T11:17:11,288 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T11:17:11,290 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T11:17:11,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T11:17:11,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T11:17:11,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:11,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:11,295 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ae52d81aa32e,36127,1731064628628, sessionid=0x101879977df0000, setting cluster-up flag (Was=false) 2024-11-08T11:17:11,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:11,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:11,313 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T11:17:11,315 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:11,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:11,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:11,326 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T11:17:11,327 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:11,401 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-08T11:17:11,407 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-08T11:17:11,409 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T11:17:11,415 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ae52d81aa32e,36127,1731064628628 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T11:17:11,418 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ae52d81aa32e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T11:17:11,418 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ae52d81aa32e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T11:17:11,418 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ae52d81aa32e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T11:17:11,418 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ae52d81aa32e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T11:17:11,418 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ae52d81aa32e:0, corePoolSize=10, maxPoolSize=10 2024-11-08T11:17:11,418 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,419 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ae52d81aa32e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T11:17:11,419 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,420 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731064661420 2024-11-08T11:17:11,421 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T11:17:11,422 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T11:17:11,423 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T11:17:11,424 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-08T11:17:11,425 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T11:17:11,425 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T11:17:11,426 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T11:17:11,426 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T11:17:11,426 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,427 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T11:17:11,428 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:11,428 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T11:17:11,428 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T11:17:11,429 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T11:17:11,431 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T11:17:11,431 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T11:17:11,433 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ae52d81aa32e:0:becomeActiveMaster-HFileCleaner.large.0-1731064631433,5,FailOnTimeoutGroup] 2024-11-08T11:17:11,433 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ae52d81aa32e:0:becomeActiveMaster-HFileCleaner.small.0-1731064631433,5,FailOnTimeoutGroup] 2024-11-08T11:17:11,433 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,434 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T11:17:11,435 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,435 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741831_1007 (size=1039) 2024-11-08T11:17:11,444 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ae52d81aa32e:33717 2024-11-08T11:17:11,446 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1008): ClusterId : c412b867-814f-4e10-ab48-e7bac993cea1 2024-11-08T11:17:11,448 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T11:17:11,453 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T11:17:11,453 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T11:17:11,456 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T11:17:11,456 DEBUG [RS:0;ae52d81aa32e:33717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db5f3b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:11,458 DEBUG [RS:0;ae52d81aa32e:33717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3ce54e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae52d81aa32e/172.17.0.2:0 2024-11-08T11:17:11,460 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-08T11:17:11,460 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-08T11:17:11,461 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-08T11:17:11,463 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3073): reportForDuty to master=ae52d81aa32e,36127,1731064628628 with isa=ae52d81aa32e/172.17.0.2:33717, startcode=1731064629394 2024-11-08T11:17:11,474 DEBUG [RS:0;ae52d81aa32e:33717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T11:17:11,509 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34805, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T11:17:11,514 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36127 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:11,517 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36127 {}] master.ServerManager(486): Registering regionserver=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:11,530 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:17:11,530 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:46645 2024-11-08T11:17:11,530 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-08T11:17:11,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T11:17:11,535 DEBUG [RS:0;ae52d81aa32e:33717 {}] zookeeper.ZKUtil(111): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:11,536 WARN [RS:0;ae52d81aa32e:33717 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T11:17:11,536 INFO [RS:0;ae52d81aa32e:33717 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T11:17:11,536 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:11,538 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae52d81aa32e,33717,1731064629394] 2024-11-08T11:17:11,552 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-08T11:17:11,567 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T11:17:11,579 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T11:17:11,582 INFO [RS:0;ae52d81aa32e:33717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T11:17:11,582 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,583 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-08T11:17:11,589 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae52d81aa32e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T11:17:11,590 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,591 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,591 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,591 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,591 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae52d81aa32e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T11:17:11,591 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae52d81aa32e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T11:17:11,591 DEBUG [RS:0;ae52d81aa32e:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae52d81aa32e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T11:17:11,592 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,592 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,592 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,592 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,593 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,33717,1731064629394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T11:17:11,612 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T11:17:11,614 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,33717,1731064629394-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:11,633 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.Replication(204): ae52d81aa32e,33717,1731064629394 started 2024-11-08T11:17:11,633 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1767): Serving as ae52d81aa32e,33717,1731064629394, RpcServer on ae52d81aa32e/172.17.0.2:33717, sessionid=0x101879977df0001 2024-11-08T11:17:11,634 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T11:17:11,634 DEBUG [RS:0;ae52d81aa32e:33717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:11,634 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae52d81aa32e,33717,1731064629394' 2024-11-08T11:17:11,634 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T11:17:11,635 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T11:17:11,635 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T11:17:11,636 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T11:17:11,636 DEBUG [RS:0;ae52d81aa32e:33717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:11,636 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae52d81aa32e,33717,1731064629394' 2024-11-08T11:17:11,636 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T11:17:11,636 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T11:17:11,637 DEBUG [RS:0;ae52d81aa32e:33717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T11:17:11,637 INFO [RS:0;ae52d81aa32e:33717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T11:17:11,637 INFO [RS:0;ae52d81aa32e:33717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T11:17:11,839 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-08T11:17:11,840 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:17:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741832_1008 (size=32) 2024-11-08T11:17:12,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:12,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T11:17:12,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T11:17:12,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:12,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:12,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T11:17:12,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T11:17:12,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:12,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:12,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T11:17:12,274 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T11:17:12,274 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:12,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:12,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740 2024-11-08T11:17:12,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740 2024-11-08T11:17:12,289 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T11:17:12,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T11:17:12,320 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:17:12,322 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69350727, jitterRate=0.03340636193752289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T11:17:12,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T11:17:12,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T11:17:12,326 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T11:17:12,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T11:17:12,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T11:17:12,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T11:17:12,334 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T11:17:12,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T11:17:12,346 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T11:17:12,346 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-08T11:17:12,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T11:17:12,366 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T11:17:12,380 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T11:17:12,531 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:13,532 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:14,533 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:15,534 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:16,535 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:17,537 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:18,538 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:19,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T11:17:19,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T11:17:19,539 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:20,539 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:21,540 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:22,541 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:23,542 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:24,542 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:25,543 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:26,544 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:27,545 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:28,546 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:29,547 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:30,547 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:31,548 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:32,549 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:33,549 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:34,550 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:35,551 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:36,551 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:37,552 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:37,770 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:17:37,966 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:17:38,552 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:39,553 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:40,554 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:41,460 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T11:17:41,463 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55214, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T11:17:41,554 WARN [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T11:17:41,643 INFO [RS:0;ae52d81aa32e:33717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T11:17:41,648 INFO [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae52d81aa32e%2C33717%2C1731064629394, suffix=, logDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394, archiveDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/oldWALs, maxLogs=32 2024-11-08T11:17:41,665 DEBUG [RS:0;ae52d81aa32e:33717 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394/ae52d81aa32e%2C33717%2C1731064629394.1731064661650, exclude list is [], retry=0 2024-11-08T11:17:41,670 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40011,DS-722f6c85-8fbf-4049-ae27-2d425b50b94f,DISK] 2024-11-08T11:17:41,673 INFO [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394/ae52d81aa32e%2C33717%2C1731064629394.1731064661650 2024-11-08T11:17:41,674 DEBUG [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34689:34689)] 2024-11-08T11:17:41,806 DEBUG [ae52d81aa32e:36127 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T11:17:41,810 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:41,816 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae52d81aa32e,33717,1731064629394, state=OPENING 2024-11-08T11:17:41,821 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T11:17:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:41,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T11:17:41,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T11:17:41,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:17:42,000 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:42,001 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T11:17:42,003 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51786, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T11:17:42,014 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-08T11:17:42,014 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T11:17:42,015 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T11:17:42,018 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae52d81aa32e%2C33717%2C1731064629394.meta, suffix=.meta, logDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394, archiveDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/oldWALs, maxLogs=32 2024-11-08T11:17:42,035 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394/ae52d81aa32e%2C33717%2C1731064629394.meta.1731064662020.meta, exclude list is [], retry=0 2024-11-08T11:17:42,038 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40011,DS-722f6c85-8fbf-4049-ae27-2d425b50b94f,DISK] 2024-11-08T11:17:42,042 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/WALs/ae52d81aa32e,33717,1731064629394/ae52d81aa32e%2C33717%2C1731064629394.meta.1731064662020.meta 2024-11-08T11:17:42,042 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34689:34689)] 2024-11-08T11:17:42,043 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:17:42,045 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T11:17:42,108 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T11:17:42,113 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T11:17:42,117 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T11:17:42,117 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:42,117 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-08T11:17:42,117 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-08T11:17:42,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T11:17:42,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T11:17:42,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:42,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:42,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T11:17:42,126 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T11:17:42,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:42,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:42,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T11:17:42,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T11:17:42,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:42,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T11:17:42,130 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740 2024-11-08T11:17:42,133 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740 2024-11-08T11:17:42,135 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T11:17:42,138 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T11:17:42,139 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69787415, jitterRate=0.039913520216941833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T11:17:42,141 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T11:17:42,148 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731064661994 2024-11-08T11:17:42,159 DEBUG [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T11:17:42,159 INFO [RS_OPEN_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-08T11:17:42,160 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:42,162 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae52d81aa32e,33717,1731064629394, state=OPEN 2024-11-08T11:17:42,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T11:17:42,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T11:17:42,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T11:17:42,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T11:17:42,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T11:17:42,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ae52d81aa32e,33717,1731064629394 in 341 msec 2024-11-08T11:17:42,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T11:17:42,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8180 sec 2024-11-08T11:17:42,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 30.8240 sec 2024-11-08T11:17:42,182 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731064662182, completionTime=-1 2024-11-08T11:17:42,183 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T11:17:42,183 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-08T11:17:42,219 DEBUG [hconnection-0x6475295a-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:42,221 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51798, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:42,230 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-08T11:17:42,230 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731064722230 2024-11-08T11:17:42,230 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731064782230 2024-11-08T11:17:42,231 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 47 msec 2024-11-08T11:17:42,251 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,36127,1731064628628-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:42,251 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,36127,1731064628628-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:42,252 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,36127,1731064628628-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:42,253 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ae52d81aa32e:36127, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:42,254 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:42,259 DEBUG [master/ae52d81aa32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-08T11:17:42,262 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-08T11:17:42,263 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T11:17:42,270 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-08T11:17:42,273 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:17:42,274 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:42,276 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:17:42,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741835_1011 (size=358) 2024-11-08T11:17:42,692 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ad75048110dcede8a9d5403d1e5644c9, NAME => 'hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:17:42,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741836_1012 (size=42) 2024-11-08T11:17:43,103 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:43,104 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing ad75048110dcede8a9d5403d1e5644c9, disabling compactions & flushes 2024-11-08T11:17:43,104 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,104 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,104 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. after waiting 0 ms 2024-11-08T11:17:43,104 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,104 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,104 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for ad75048110dcede8a9d5403d1e5644c9: 2024-11-08T11:17:43,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:17:43,113 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731064663108"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064663108"}]},"ts":"1731064663108"} 2024-11-08T11:17:43,140 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:17:43,142 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:17:43,145 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064663143"}]},"ts":"1731064663143"} 2024-11-08T11:17:43,150 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-08T11:17:43,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=ad75048110dcede8a9d5403d1e5644c9, ASSIGN}] 2024-11-08T11:17:43,158 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=ad75048110dcede8a9d5403d1e5644c9, ASSIGN 2024-11-08T11:17:43,160 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=ad75048110dcede8a9d5403d1e5644c9, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:17:43,311 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=ad75048110dcede8a9d5403d1e5644c9, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:43,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure ad75048110dcede8a9d5403d1e5644c9, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:17:43,469 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:43,475 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,475 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => ad75048110dcede8a9d5403d1e5644c9, NAME => 'hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:17:43,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:43,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,479 INFO [StoreOpener-ad75048110dcede8a9d5403d1e5644c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,481 INFO [StoreOpener-ad75048110dcede8a9d5403d1e5644c9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad75048110dcede8a9d5403d1e5644c9 columnFamilyName info 2024-11-08T11:17:43,481 DEBUG [StoreOpener-ad75048110dcede8a9d5403d1e5644c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:43,482 INFO [StoreOpener-ad75048110dcede8a9d5403d1e5644c9-1 {}] regionserver.HStore(327): Store=ad75048110dcede8a9d5403d1e5644c9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:43,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,487 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:17:43,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:17:43,491 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened ad75048110dcede8a9d5403d1e5644c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68852050, jitterRate=0.02597549557685852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:17:43,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for ad75048110dcede8a9d5403d1e5644c9: 2024-11-08T11:17:43,495 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9., pid=6, masterSystemTime=1731064663469 2024-11-08T11:17:43,498 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,498 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:17:43,499 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=ad75048110dcede8a9d5403d1e5644c9, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:43,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T11:17:43,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure ad75048110dcede8a9d5403d1e5644c9, server=ae52d81aa32e,33717,1731064629394 in 187 msec 2024-11-08T11:17:43,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T11:17:43,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=ad75048110dcede8a9d5403d1e5644c9, ASSIGN in 350 msec 2024-11-08T11:17:43,512 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:17:43,512 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064663512"}]},"ts":"1731064663512"} 2024-11-08T11:17:43,515 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-08T11:17:43,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:17:43,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2540 sec 2024-11-08T11:17:43,574 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-08T11:17:43,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:43,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-08T11:17:43,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:17:43,605 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-08T11:17:43,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T11:17:43,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 23 msec 2024-11-08T11:17:43,628 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-08T11:17:43,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T11:17:43,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2024-11-08T11:17:43,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-08T11:17:43,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-08T11:17:43,658 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 34.194sec 2024-11-08T11:17:43,660 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T11:17:43,661 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T11:17:43,662 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T11:17:43,663 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T11:17:43,663 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T11:17:43,664 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,36127,1731064628628-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T11:17:43,664 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,36127,1731064628628-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T11:17:43,675 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-08T11:17:43,676 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T11:17:43,676 INFO [master/ae52d81aa32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae52d81aa32e,36127,1731064628628-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T11:17:43,679 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x574a453f to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61e4a9fc 2024-11-08T11:17:43,679 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-08T11:17:43,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d933e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:43,691 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T11:17:43,691 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T11:17:43,704 DEBUG [hconnection-0x35085285-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:43,717 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51802, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:43,732 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ae52d81aa32e,36127,1731064628628 2024-11-08T11:17:43,751 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=2970 2024-11-08T11:17:43,757 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x312a75fa to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42bb6992 2024-11-08T11:17:43,762 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x312a75fa {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb18e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:43,767 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x144b7569 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d25698b 2024-11-08T11:17:43,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ae8247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:43,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T11:17:43,781 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55218, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T11:17:43,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:17:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T11:17:43,794 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:17:43,795 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:43,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:17:43,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-08T11:17:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T11:17:43,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741837_1013 (size=437) 2024-11-08T11:17:44,210 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2ae2b67ac99527ab93cd9d51d12df9c3, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:17:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741838_1014 (size=88) 2024-11-08T11:17:44,620 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:44,620 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2ae2b67ac99527ab93cd9d51d12df9c3, disabling compactions & flushes 2024-11-08T11:17:44,620 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,620 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,620 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. after waiting 0 ms 2024-11-08T11:17:44,621 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,621 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,621 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2ae2b67ac99527ab93cd9d51d12df9c3: 2024-11-08T11:17:44,623 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:17:44,623 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731064664623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064664623"}]},"ts":"1731064664623"} 2024-11-08T11:17:44,626 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:17:44,628 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:17:44,628 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064664628"}]},"ts":"1731064664628"} 2024-11-08T11:17:44,630 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:17:44,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=2ae2b67ac99527ab93cd9d51d12df9c3, ASSIGN}] 2024-11-08T11:17:44,636 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=2ae2b67ac99527ab93cd9d51d12df9c3, ASSIGN 2024-11-08T11:17:44,638 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=2ae2b67ac99527ab93cd9d51d12df9c3, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:17:44,788 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=2ae2b67ac99527ab93cd9d51d12df9c3, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:44,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 2ae2b67ac99527ab93cd9d51d12df9c3, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:17:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T11:17:44,946 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:44,952 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,953 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 2ae2b67ac99527ab93cd9d51d12df9c3, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:17:44,953 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,953 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:44,953 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,954 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,956 INFO [StoreOpener-2ae2b67ac99527ab93cd9d51d12df9c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,958 INFO [StoreOpener-2ae2b67ac99527ab93cd9d51d12df9c3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ae2b67ac99527ab93cd9d51d12df9c3 columnFamilyName testFamily 2024-11-08T11:17:44,958 DEBUG [StoreOpener-2ae2b67ac99527ab93cd9d51d12df9c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:44,959 INFO [StoreOpener-2ae2b67ac99527ab93cd9d51d12df9c3-1 {}] regionserver.HStore(327): Store=2ae2b67ac99527ab93cd9d51d12df9c3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:44,961 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,961 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,965 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:17:44,969 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:17:44,970 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 2ae2b67ac99527ab93cd9d51d12df9c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66034250, jitterRate=-0.016012996435165405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:17:44,972 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 2ae2b67ac99527ab93cd9d51d12df9c3: 2024-11-08T11:17:44,992 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3., pid=11, masterSystemTime=1731064664946 2024-11-08T11:17:44,996 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,996 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:17:44,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=2ae2b67ac99527ab93cd9d51d12df9c3, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:45,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T11:17:45,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 2ae2b67ac99527ab93cd9d51d12df9c3, server=ae52d81aa32e,33717,1731064629394 in 208 msec 2024-11-08T11:17:45,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T11:17:45,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=2ae2b67ac99527ab93cd9d51d12df9c3, ASSIGN in 369 msec 2024-11-08T11:17:45,008 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:17:45,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064665008"}]},"ts":"1731064665008"} 2024-11-08T11:17:45,011 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:17:45,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:17:45,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 1.2240 sec 2024-11-08T11:17:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T11:17:46,813 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-08T11:17:46,813 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:17:46,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:17:46,821 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T11:17:46,821 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:17:46,822 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-08T11:17:46,825 DEBUG [hconnection-0x6b471694-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:46,830 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51810, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:46,852 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:17:46,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T11:17:46,880 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:17:46,882 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x312a75fa {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:17:46,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:46,891 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51826, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:17:46,934 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T11:17:46,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300043 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:46,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T11:17:46,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300043 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:46,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300043 2024-11-08T11:17:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51826 deadline: 1731064669937, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:17:46,976 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300043 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:17:46,979 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T11:17:46,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300044 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:46,982 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T11:17:46,982 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:17:46,986 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:17:46,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300045 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:46,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300045 2024-11-08T11:17:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51826 deadline: 1731064669986, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:17:46,989 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300045 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:17:46,990 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:17:46,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300046 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:46,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4132839332912300046 2024-11-08T11:17:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51826 deadline: 1731064669990, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:17:46,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:17:46,992 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300046 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:17:46,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300047 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:46,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4132839332912300047 2024-11-08T11:17:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51826 deadline: 1731064669994, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:17:46,998 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300047 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:17:47,000 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:17:47,002 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x144b7569 to 127.0.0.1:55518 2024-11-08T11:17:47,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:17:47,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:17:47,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:17:47,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:17:47,009 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x312a75fa to 127.0.0.1:55518 2024-11-08T11:17:47,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=226 (was 220) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55518@0x144b7569 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$738/0x00007f7154b46430.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55518@0x312a75fa-SendThread(127.0.0.1:55518) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: hconnection-0x35085285-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55518@0x312a75fa-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55518@0x144b7569-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55518@0x144b7569-SendThread(127.0.0.1:55518) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2036388686_22 at /127.0.0.1:35286 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55518@0x312a75fa java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$738/0x00007f7154b46430.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 273) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2906 (was 2970) 2024-11-08T11:17:47,043 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=226, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=2905 2024-11-08T11:17:47,046 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e567203 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@488bbf84 2024-11-08T11:17:47,053 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x3e567203 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a44599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:47,055 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b4e4ad2 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1760f201 2024-11-08T11:17:47,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1648742b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:47,075 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:17:47,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T11:17:47,079 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:17:47,079 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:47,080 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-08T11:17:47,081 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:17:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T11:17:47,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741839_1015 (size=423) 2024-11-08T11:17:47,096 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a198a6c470d9a189e91f0680bf15f8f4, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:17:47,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741840_1016 (size=74) 2024-11-08T11:17:47,530 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:47,530 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing a198a6c470d9a189e91f0680bf15f8f4, disabling compactions & flushes 2024-11-08T11:17:47,530 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,530 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,530 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. after waiting 0 ms 2024-11-08T11:17:47,530 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,530 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,530 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for a198a6c470d9a189e91f0680bf15f8f4: 2024-11-08T11:17:47,533 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:17:47,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731064667534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064667534"}]},"ts":"1731064667534"} 2024-11-08T11:17:47,545 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:17:47,547 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:17:47,548 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064667548"}]},"ts":"1731064667548"} 2024-11-08T11:17:47,557 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:17:47,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=a198a6c470d9a189e91f0680bf15f8f4, ASSIGN}] 2024-11-08T11:17:47,565 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=a198a6c470d9a189e91f0680bf15f8f4, ASSIGN 2024-11-08T11:17:47,567 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=a198a6c470d9a189e91f0680bf15f8f4, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:17:47,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:17:47,715 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-08T11:17:47,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T11:17:47,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-08T11:17:47,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=a198a6c470d9a189e91f0680bf15f8f4, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:47,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure a198a6c470d9a189e91f0680bf15f8f4, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:17:47,877 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:47,884 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => a198a6c470d9a189e91f0680bf15f8f4, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:17:47,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:47,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,887 INFO [StoreOpener-a198a6c470d9a189e91f0680bf15f8f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,894 INFO [StoreOpener-a198a6c470d9a189e91f0680bf15f8f4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a198a6c470d9a189e91f0680bf15f8f4 columnFamilyName testFamily 2024-11-08T11:17:47,894 DEBUG [StoreOpener-a198a6c470d9a189e91f0680bf15f8f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:47,895 INFO [StoreOpener-a198a6c470d9a189e91f0680bf15f8f4-1 {}] regionserver.HStore(327): Store=a198a6c470d9a189e91f0680bf15f8f4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:47,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,902 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:17:47,907 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:17:47,908 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened a198a6c470d9a189e91f0680bf15f8f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67950127, jitterRate=0.01253579556941986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:17:47,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for a198a6c470d9a189e91f0680bf15f8f4: 2024-11-08T11:17:47,911 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4., pid=14, masterSystemTime=1731064667876 2024-11-08T11:17:47,916 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=a198a6c470d9a189e91f0680bf15f8f4, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:47,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T11:17:47,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure a198a6c470d9a189e91f0680bf15f8f4, server=ae52d81aa32e,33717,1731064629394 in 196 msec 2024-11-08T11:17:47,928 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,928 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:17:47,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-08T11:17:47,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=a198a6c470d9a189e91f0680bf15f8f4, ASSIGN in 363 msec 2024-11-08T11:17:47,931 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:17:47,931 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064667931"}]},"ts":"1731064667931"} 2024-11-08T11:17:47,934 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:17:47,939 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:17:47,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 864 msec 2024-11-08T11:17:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T11:17:48,086 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-08T11:17:48,087 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:17:48,087 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:17:48,094 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T11:17:48,094 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:17:48,094 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T11:17:48,102 DEBUG [hconnection-0x512e1248-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:48,114 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51842, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:48,137 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:17:48,142 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:17:48,149 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:17:49,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T11:17:49,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:17:49,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T11:17:49,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T11:17:49,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T11:17:49,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-08T11:17:49,129 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T11:17:49,129 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T11:17:49,129 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T11:17:49,129 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:17:53,387 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:17:53,720 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-08T11:17:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T11:17:55,180 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:17:57,148 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:17:57,149 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T11:17:57.148Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1968ms, rpcTimeout=1965ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:17:57,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-08T11:17:57,151 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b4e4ad2 to 127.0.0.1:55518 2024-11-08T11:17:57,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:17:57,151 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:17:57,151 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:17:57,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:17:57,151 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3e567203 to 127.0.0.1:55518 2024-11-08T11:17:57,166 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=225 (was 226), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 339) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2838 (was 2905) 2024-11-08T11:17:57,175 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=2838 2024-11-08T11:17:57,178 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b8ee3d4 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51fb2da6 2024-11-08T11:17:57,182 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x2b8ee3d4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bd8c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:57,183 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c55ebef to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e7c030c 2024-11-08T11:17:57,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2230cf54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:17:57,190 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:17:57,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T11:17:57,193 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:17:57,193 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:57,194 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-08T11:17:57,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:17:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T11:17:57,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741841_1017 (size=421) 2024-11-08T11:17:57,608 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 27c4e829c2cf184f5508de640a25b0bb, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:17:57,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741842_1018 (size=72) 2024-11-08T11:17:57,619 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:57,619 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 27c4e829c2cf184f5508de640a25b0bb, disabling compactions & flushes 2024-11-08T11:17:57,619 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,619 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,619 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. after waiting 0 ms 2024-11-08T11:17:57,619 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,619 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,619 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 27c4e829c2cf184f5508de640a25b0bb: 2024-11-08T11:17:57,621 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:17:57,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731064677621"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064677621"}]},"ts":"1731064677621"} 2024-11-08T11:17:57,624 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:17:57,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:17:57,626 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064677626"}]},"ts":"1731064677626"} 2024-11-08T11:17:57,629 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:17:57,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=27c4e829c2cf184f5508de640a25b0bb, ASSIGN}] 2024-11-08T11:17:57,636 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=27c4e829c2cf184f5508de640a25b0bb, ASSIGN 2024-11-08T11:17:57,637 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=27c4e829c2cf184f5508de640a25b0bb, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:17:57,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=27c4e829c2cf184f5508de640a25b0bb, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:57,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 27c4e829c2cf184f5508de640a25b0bb, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:17:57,944 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:57,951 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,951 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 27c4e829c2cf184f5508de640a25b0bb, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:17:57,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:17:57,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,954 INFO [StoreOpener-27c4e829c2cf184f5508de640a25b0bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,956 INFO [StoreOpener-27c4e829c2cf184f5508de640a25b0bb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27c4e829c2cf184f5508de640a25b0bb columnFamilyName testFamily 2024-11-08T11:17:57,957 DEBUG [StoreOpener-27c4e829c2cf184f5508de640a25b0bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:17:57,958 INFO [StoreOpener-27c4e829c2cf184f5508de640a25b0bb-1 {}] regionserver.HStore(327): Store=27c4e829c2cf184f5508de640a25b0bb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:17:57,959 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,960 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,965 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:17:57,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:17:57,971 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened 27c4e829c2cf184f5508de640a25b0bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63748735, jitterRate=-0.05006982386112213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:17:57,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 27c4e829c2cf184f5508de640a25b0bb: 2024-11-08T11:17:57,975 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb., pid=17, masterSystemTime=1731064677944 2024-11-08T11:17:57,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=27c4e829c2cf184f5508de640a25b0bb, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:17:57,990 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,990 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:17:57,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-08T11:17:57,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 27c4e829c2cf184f5508de640a25b0bb, server=ae52d81aa32e,33717,1731064629394 in 201 msec 2024-11-08T11:17:58,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T11:17:58,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=27c4e829c2cf184f5508de640a25b0bb, ASSIGN in 363 msec 2024-11-08T11:17:58,002 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:17:58,003 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064678003"}]},"ts":"1731064678003"} 2024-11-08T11:17:58,006 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:17:58,011 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:17:58,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 821 msec 2024-11-08T11:17:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T11:17:58,198 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-08T11:17:58,199 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:17:58,199 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:17:58,203 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T11:17:58,204 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:17:58,204 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T11:17:58,206 DEBUG [hconnection-0x6343ed0b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:58,208 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47874, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:58,221 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:17:58,225 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:17:58,225 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:17:58,227 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x2b8ee3d4 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:17:58,227 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x2b8ee3d4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:17:58,229 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47878, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:17:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:17:58,232 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:17:58,233 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300059 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:58,234 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:17:58,234 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300059 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:17:58,235 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T11:17:59,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T11:17:59,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:17:59,721 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-08T11:18:01,681 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:03,455 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:07,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9003 2024-11-08T11:18:07,238 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c55ebef to 127.0.0.1:55518 2024-11-08T11:18:07,238 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300059 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:18:07,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:07,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:07,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300059' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T11:18:07,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:07,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:47878 deadline: 1731064693237, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300059' does not exist 2024-11-08T11:18:07,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:07,239 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b8ee3d4 to 127.0.0.1:55518 2024-11-08T11:18:07,239 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:07,239 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ae52d81aa32e,33717,1731064629394 for closing scanner id = 4132839332912300059 for 27c4e829c2cf184f5508de640a25b0bb of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T11:18:07,249 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=225 (was 225), OpenFileDescriptor=452 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=338 (was 348), ProcessCount=11 (was 11), AvailableMemoryMB=2764 (was 2838) 2024-11-08T11:18:07,257 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=225, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=2764 2024-11-08T11:18:07,259 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b6678da to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58325c54 2024-11-08T11:18:07,262 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x1b6678da {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b1de20e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:07,263 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d914229 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@142c43db 2024-11-08T11:18:07,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca4492c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:07,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T11:18:07,273 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:07,273 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:07,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-08T11:18:07,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T11:18:07,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741843_1019 (size=432) 2024-11-08T11:18:07,686 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8c25b44c431bf3f90b6b41bc98a7e29d, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:07,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741844_1020 (size=83) 2024-11-08T11:18:07,736 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:07,771 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:18:08,096 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:08,096 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 8c25b44c431bf3f90b6b41bc98a7e29d, disabling compactions & flushes 2024-11-08T11:18:08,096 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,096 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,096 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. after waiting 0 ms 2024-11-08T11:18:08,096 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,096 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,096 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8c25b44c431bf3f90b6b41bc98a7e29d: 2024-11-08T11:18:08,098 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:08,098 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731064688098"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064688098"}]},"ts":"1731064688098"} 2024-11-08T11:18:08,101 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:08,102 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:08,102 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064688102"}]},"ts":"1731064688102"} 2024-11-08T11:18:08,104 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:08,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=8c25b44c431bf3f90b6b41bc98a7e29d, ASSIGN}] 2024-11-08T11:18:08,113 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=8c25b44c431bf3f90b6b41bc98a7e29d, ASSIGN 2024-11-08T11:18:08,114 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=8c25b44c431bf3f90b6b41bc98a7e29d, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:08,264 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=8c25b44c431bf3f90b6b41bc98a7e29d, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:08,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 8c25b44c431bf3f90b6b41bc98a7e29d, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T11:18:08,420 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:08,429 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,429 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => 8c25b44c431bf3f90b6b41bc98a7e29d, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:08,429 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,429 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:08,429 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for 8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,430 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for 8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,432 INFO [StoreOpener-8c25b44c431bf3f90b6b41bc98a7e29d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,434 INFO [StoreOpener-8c25b44c431bf3f90b6b41bc98a7e29d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c25b44c431bf3f90b6b41bc98a7e29d columnFamilyName testFamily 2024-11-08T11:18:08,434 DEBUG [StoreOpener-8c25b44c431bf3f90b6b41bc98a7e29d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:08,435 INFO [StoreOpener-8c25b44c431bf3f90b6b41bc98a7e29d-1 {}] regionserver.HStore(327): Store=8c25b44c431bf3f90b6b41bc98a7e29d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:08,436 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,437 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,440 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for 8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:18:08,444 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:08,445 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened 8c25b44c431bf3f90b6b41bc98a7e29d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60867130, jitterRate=-0.0930090844631195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:08,446 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for 8c25b44c431bf3f90b6b41bc98a7e29d: 2024-11-08T11:18:08,447 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d., pid=20, masterSystemTime=1731064688420 2024-11-08T11:18:08,450 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,450 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:18:08,451 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=8c25b44c431bf3f90b6b41bc98a7e29d, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:08,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T11:18:08,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 8c25b44c431bf3f90b6b41bc98a7e29d, server=ae52d81aa32e,33717,1731064629394 in 187 msec 2024-11-08T11:18:08,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-08T11:18:08,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=8c25b44c431bf3f90b6b41bc98a7e29d, ASSIGN in 346 msec 2024-11-08T11:18:08,462 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:08,462 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064688462"}]},"ts":"1731064688462"} 2024-11-08T11:18:08,464 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:08,468 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:08,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.1980 sec 2024-11-08T11:18:09,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T11:18:09,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T11:18:10,279 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-08T11:18:10,279 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:10,279 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:10,284 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:10,284 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:10,284 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-08T11:18:10,286 DEBUG [hconnection-0x5ce542a2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:10,288 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46542, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:10,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:10,301 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T11:18:10,303 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T11:18:10,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300063 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:10,305 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T11:18:10,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300063 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:10,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300063 2024-11-08T11:18:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46542 deadline: 1731064691305, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:18:10,307 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:18:10,309 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T11:18:10,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300064 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:10,310 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T11:18:10,310 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:18:10,312 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:18:10,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300065 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:10,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300065 2024-11-08T11:18:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46542 deadline: 1731064691312, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:18:10,313 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:18:10,315 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:18:10,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300066 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:10,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4132839332912300066 2024-11-08T11:18:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46542 deadline: 1731064691315, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:18:10,316 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:18:10,318 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:18:10,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300067 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:10,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4132839332912300067 2024-11-08T11:18:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46542 deadline: 1731064691318, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:18:10,319 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:18:10,320 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:18:10,321 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d914229 to 127.0.0.1:55518 2024-11-08T11:18:10,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:10,321 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:10,321 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:10,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:10,321 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b6678da to 127.0.0.1:55518 2024-11-08T11:18:10,330 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=226 (was 225) - Thread LEAK? -, OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=338 (was 338), ProcessCount=11 (was 11), AvailableMemoryMB=2751 (was 2764) 2024-11-08T11:18:10,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=2750 2024-11-08T11:18:10,341 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66b76e83 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a9c4477 2024-11-08T11:18:10,344 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x66b76e83 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e863530, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:10,345 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x428a041b to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d52f25d 2024-11-08T11:18:10,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50cc5602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:10,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T11:18:10,355 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:10,355 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:10,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-08T11:18:10,356 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T11:18:10,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741845_1021 (size=426) 2024-11-08T11:18:10,767 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => decbab74a7be3e3a5b25bee2e10ab413, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:10,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741846_1022 (size=77) 2024-11-08T11:18:11,175 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:11,176 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing decbab74a7be3e3a5b25bee2e10ab413, disabling compactions & flushes 2024-11-08T11:18:11,176 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,176 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,176 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. after waiting 0 ms 2024-11-08T11:18:11,176 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,176 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,176 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for decbab74a7be3e3a5b25bee2e10ab413: 2024-11-08T11:18:11,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:11,178 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731064691178"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064691178"}]},"ts":"1731064691178"} 2024-11-08T11:18:11,181 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:11,182 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:11,182 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064691182"}]},"ts":"1731064691182"} 2024-11-08T11:18:11,184 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:11,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=decbab74a7be3e3a5b25bee2e10ab413, ASSIGN}] 2024-11-08T11:18:11,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=decbab74a7be3e3a5b25bee2e10ab413, ASSIGN 2024-11-08T11:18:11,190 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=decbab74a7be3e3a5b25bee2e10ab413, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:11,341 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=decbab74a7be3e3a5b25bee2e10ab413, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:11,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure decbab74a7be3e3a5b25bee2e10ab413, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T11:18:11,497 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:11,502 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,502 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => decbab74a7be3e3a5b25bee2e10ab413, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:11,503 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,503 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:11,503 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,503 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,505 INFO [StoreOpener-decbab74a7be3e3a5b25bee2e10ab413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,508 INFO [StoreOpener-decbab74a7be3e3a5b25bee2e10ab413-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region decbab74a7be3e3a5b25bee2e10ab413 columnFamilyName testFamily 2024-11-08T11:18:11,508 DEBUG [StoreOpener-decbab74a7be3e3a5b25bee2e10ab413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:11,508 INFO [StoreOpener-decbab74a7be3e3a5b25bee2e10ab413-1 {}] regionserver.HStore(327): Store=decbab74a7be3e3a5b25bee2e10ab413/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:11,509 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,510 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,512 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:18:11,515 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:11,516 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened decbab74a7be3e3a5b25bee2e10ab413; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60793152, jitterRate=-0.09411144256591797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:11,516 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for decbab74a7be3e3a5b25bee2e10ab413: 2024-11-08T11:18:11,518 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413., pid=23, masterSystemTime=1731064691497 2024-11-08T11:18:11,520 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,520 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:18:11,521 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=decbab74a7be3e3a5b25bee2e10ab413, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:11,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-08T11:18:11,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure decbab74a7be3e3a5b25bee2e10ab413, server=ae52d81aa32e,33717,1731064629394 in 179 msec 2024-11-08T11:18:11,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T11:18:11,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=decbab74a7be3e3a5b25bee2e10ab413, ASSIGN in 338 msec 2024-11-08T11:18:11,530 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:11,530 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064691530"}]},"ts":"1731064691530"} 2024-11-08T11:18:11,532 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:11,535 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:11,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.1830 sec 2024-11-08T11:18:11,670 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:18:11,723 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-08T11:18:11,723 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-08T11:18:12,285 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:18:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T11:18:13,360 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-08T11:18:13,360 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:13,361 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:13,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:13,365 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:13,366 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T11:18:13,368 DEBUG [hconnection-0x72478302-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:13,370 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46544, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:13,380 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:13,382 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:18:13,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:18:13,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T11:18:13,931 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T11:18:15,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46544 deadline: 1731064696385, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:18:15,401 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:18:15,402 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300072 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:18:15,402 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2020 2024-11-08T11:18:15,403 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x428a041b to 127.0.0.1:55518 2024-11-08T11:18:15,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:15,403 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:15,403 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:15,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:15,404 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66b76e83 to 127.0.0.1:55518 2024-11-08T11:18:15,415 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=225 (was 226), OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 338), ProcessCount=11 (was 11), AvailableMemoryMB=2761 (was 2750) - AvailableMemoryMB LEAK? - 2024-11-08T11:18:15,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=2761 2024-11-08T11:18:15,429 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69ad79a5 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69e88404 2024-11-08T11:18:15,433 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x69ad79a5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61678382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:15,435 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3efb3040 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44175fc0 2024-11-08T11:18:15,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c8e56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:15,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T11:18:15,446 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:15,446 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:15,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-08T11:18:15,448 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T11:18:15,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741847_1023 (size=427) 2024-11-08T11:18:15,462 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e187798f3f2dc14ef76d302202186f7c, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:15,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741848_1024 (size=78) 2024-11-08T11:18:15,484 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:15,484 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing e187798f3f2dc14ef76d302202186f7c, disabling compactions & flushes 2024-11-08T11:18:15,485 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. after waiting 0 ms 2024-11-08T11:18:15,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,485 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for e187798f3f2dc14ef76d302202186f7c: 2024-11-08T11:18:15,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:15,487 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731064695487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064695487"}]},"ts":"1731064695487"} 2024-11-08T11:18:15,490 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:15,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:15,491 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064695491"}]},"ts":"1731064695491"} 2024-11-08T11:18:15,494 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:15,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=e187798f3f2dc14ef76d302202186f7c, ASSIGN}] 2024-11-08T11:18:15,500 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=e187798f3f2dc14ef76d302202186f7c, ASSIGN 2024-11-08T11:18:15,502 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=e187798f3f2dc14ef76d302202186f7c, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:15,652 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=e187798f3f2dc14ef76d302202186f7c, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:15,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure e187798f3f2dc14ef76d302202186f7c, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:15,809 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:15,815 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,815 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => e187798f3f2dc14ef76d302202186f7c, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:15,816 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,816 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:15,816 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,816 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,836 INFO [StoreOpener-e187798f3f2dc14ef76d302202186f7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,839 INFO [StoreOpener-e187798f3f2dc14ef76d302202186f7c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e187798f3f2dc14ef76d302202186f7c columnFamilyName testFamily 2024-11-08T11:18:15,840 DEBUG [StoreOpener-e187798f3f2dc14ef76d302202186f7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:15,840 INFO [StoreOpener-e187798f3f2dc14ef76d302202186f7c-1 {}] regionserver.HStore(327): Store=e187798f3f2dc14ef76d302202186f7c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:15,842 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,843 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,847 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:18:15,851 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:15,852 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened e187798f3f2dc14ef76d302202186f7c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75231891, jitterRate=0.12104253470897675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:15,854 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for e187798f3f2dc14ef76d302202186f7c: 2024-11-08T11:18:15,856 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c., pid=26, masterSystemTime=1731064695808 2024-11-08T11:18:15,860 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=e187798f3f2dc14ef76d302202186f7c, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:15,861 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:18:15,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T11:18:15,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure e187798f3f2dc14ef76d302202186f7c, server=ae52d81aa32e,33717,1731064629394 in 209 msec 2024-11-08T11:18:15,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T11:18:15,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=e187798f3f2dc14ef76d302202186f7c, ASSIGN in 370 msec 2024-11-08T11:18:15,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:15,875 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064695875"}]},"ts":"1731064695875"} 2024-11-08T11:18:15,878 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:15,881 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:15,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 439 msec 2024-11-08T11:18:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T11:18:16,452 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-08T11:18:16,452 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:16,453 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:16,459 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:16,459 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:16,459 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T11:18:16,462 DEBUG [hconnection-0xa1f1e4c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:16,465 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46556, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:16,479 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:16,483 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:18:16,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:18:17,725 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-08T11:18:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T11:18:18,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:18:19,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T11:18:19,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:19,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T11:18:19,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:19,487 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:18:19,488 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T11:18:19.488Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:18:19,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2024-11-08T11:18:19,489 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3efb3040 to 127.0.0.1:55518 2024-11-08T11:18:19,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:19,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:19,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:19,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:19,490 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69ad79a5 to 127.0.0.1:55518 2024-11-08T11:18:19,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=226 (was 225) - Thread LEAK? -, OpenFileDescriptor=456 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 311) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2744 (was 2761) 2024-11-08T11:18:19,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=226, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=2743 2024-11-08T11:18:19,514 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50ca2cd2 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29e3c0fe 2024-11-08T11:18:19,518 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x50ca2cd2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f51a389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:19,520 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4dd486c6 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a698487 2024-11-08T11:18:19,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b449ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:19,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T11:18:19,529 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:19,529 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:19,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-08T11:18:19,530 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T11:18:19,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741849_1025 (size=425) 2024-11-08T11:18:19,544 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a97e8684ef3557cc5c8111e42191ba0f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:19,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741850_1026 (size=76) 2024-11-08T11:18:19,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:19,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing a97e8684ef3557cc5c8111e42191ba0f, disabling compactions & flushes 2024-11-08T11:18:19,960 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:19,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:19,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. after waiting 0 ms 2024-11-08T11:18:19,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:19,960 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:19,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for a97e8684ef3557cc5c8111e42191ba0f: 2024-11-08T11:18:19,962 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:19,962 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731064699962"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064699962"}]},"ts":"1731064699962"} 2024-11-08T11:18:19,965 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:19,967 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:19,967 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064699967"}]},"ts":"1731064699967"} 2024-11-08T11:18:19,969 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:19,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=a97e8684ef3557cc5c8111e42191ba0f, ASSIGN}] 2024-11-08T11:18:19,975 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=a97e8684ef3557cc5c8111e42191ba0f, ASSIGN 2024-11-08T11:18:19,976 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=a97e8684ef3557cc5c8111e42191ba0f, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:20,003 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:20,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=a97e8684ef3557cc5c8111e42191ba0f, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:20,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure a97e8684ef3557cc5c8111e42191ba0f, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:20,284 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:20,290 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:20,291 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => a97e8684ef3557cc5c8111e42191ba0f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:20,291 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,291 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:20,292 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,292 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,294 INFO [StoreOpener-a97e8684ef3557cc5c8111e42191ba0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,297 INFO [StoreOpener-a97e8684ef3557cc5c8111e42191ba0f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a97e8684ef3557cc5c8111e42191ba0f columnFamilyName testFamily 2024-11-08T11:18:20,297 DEBUG [StoreOpener-a97e8684ef3557cc5c8111e42191ba0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:20,298 INFO [StoreOpener-a97e8684ef3557cc5c8111e42191ba0f-1 {}] regionserver.HStore(327): Store=a97e8684ef3557cc5c8111e42191ba0f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:20,299 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,300 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,305 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:18:20,308 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:20,309 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened a97e8684ef3557cc5c8111e42191ba0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61277841, jitterRate=-0.08688901364803314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:20,310 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for a97e8684ef3557cc5c8111e42191ba0f: 2024-11-08T11:18:20,312 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f., pid=29, masterSystemTime=1731064700284 2024-11-08T11:18:20,314 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:20,314 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:18:20,315 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=a97e8684ef3557cc5c8111e42191ba0f, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:20,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T11:18:20,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure a97e8684ef3557cc5c8111e42191ba0f, server=ae52d81aa32e,33717,1731064629394 in 188 msec 2024-11-08T11:18:20,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-08T11:18:20,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=a97e8684ef3557cc5c8111e42191ba0f, ASSIGN in 348 msec 2024-11-08T11:18:20,325 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:20,325 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064700325"}]},"ts":"1731064700325"} 2024-11-08T11:18:20,327 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:20,330 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:20,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 804 msec 2024-11-08T11:18:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T11:18:20,534 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-08T11:18:20,534 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:20,534 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:20,540 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:20,540 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:20,541 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T11:18:20,543 DEBUG [hconnection-0x4ca1d81b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:20,545 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54060, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:20,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:20,560 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:18:20,560 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:18:20,562 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x50ca2cd2 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:18:20,562 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x50ca2cd2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:20,566 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54068, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:18:20,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:18:20,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300084 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:20,570 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:18:20,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300084 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:20,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T11:18:21,318 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:23,578 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3007 2024-11-08T11:18:23,578 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4dd486c6 to 127.0.0.1:55518 2024-11-08T11:18:23,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:23,578 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:23,578 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300084 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:18:23,578 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:23,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:23,578 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50ca2cd2 to 127.0.0.1:55518 2024-11-08T11:18:23,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300084' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T11:18:23,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:54068 deadline: 1731064704578, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300084' does not exist 2024-11-08T11:18:23,579 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ae52d81aa32e,33717,1731064629394 for closing scanner id = 4132839332912300084 for a97e8684ef3557cc5c8111e42191ba0f of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T11:18:23,582 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:23,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=228 (was 226) - Thread LEAK? -, OpenFileDescriptor=456 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 334) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2718 (was 2743) 2024-11-08T11:18:23,625 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=2714 2024-11-08T11:18:23,628 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1a1dbdc4 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d75d0eb 2024-11-08T11:18:23,632 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x1a1dbdc4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f62eaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:23,634 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4feba97d to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1344acba 2024-11-08T11:18:23,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c0d08b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:23,649 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:23,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-08T11:18:23,652 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:23,652 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:23,653 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-08T11:18:23,654 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T11:18:23,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741851_1027 (size=417) 2024-11-08T11:18:23,726 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-08T11:18:24,068 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0aa76efa23785cc8435444278f71e915, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:24,072 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:24,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741852_1028 (size=68) 2024-11-08T11:18:24,089 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:24,089 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 0aa76efa23785cc8435444278f71e915, disabling compactions & flushes 2024-11-08T11:18:24,089 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,089 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,089 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. after waiting 0 ms 2024-11-08T11:18:24,089 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,090 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0aa76efa23785cc8435444278f71e915: 2024-11-08T11:18:24,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:24,092 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731064704092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064704092"}]},"ts":"1731064704092"} 2024-11-08T11:18:24,095 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:24,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:24,096 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064704096"}]},"ts":"1731064704096"} 2024-11-08T11:18:24,099 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:24,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=0aa76efa23785cc8435444278f71e915, ASSIGN}] 2024-11-08T11:18:24,105 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=0aa76efa23785cc8435444278f71e915, ASSIGN 2024-11-08T11:18:24,106 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=0aa76efa23785cc8435444278f71e915, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:24,257 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=0aa76efa23785cc8435444278f71e915, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:24,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 0aa76efa23785cc8435444278f71e915, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:24,413 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:24,420 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,420 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 0aa76efa23785cc8435444278f71e915, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:24,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:24,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,425 INFO [StoreOpener-0aa76efa23785cc8435444278f71e915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,427 INFO [master/ae52d81aa32e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T11:18:24,427 INFO [master/ae52d81aa32e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T11:18:24,428 INFO [StoreOpener-0aa76efa23785cc8435444278f71e915-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0aa76efa23785cc8435444278f71e915 columnFamilyName testFamily 2024-11-08T11:18:24,428 DEBUG [StoreOpener-0aa76efa23785cc8435444278f71e915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:24,429 INFO [StoreOpener-0aa76efa23785cc8435444278f71e915-1 {}] regionserver.HStore(327): Store=0aa76efa23785cc8435444278f71e915/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:24,430 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,431 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,434 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 0aa76efa23785cc8435444278f71e915 2024-11-08T11:18:24,438 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:24,439 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 0aa76efa23785cc8435444278f71e915; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69026811, jitterRate=0.02857963740825653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:24,440 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 0aa76efa23785cc8435444278f71e915: 2024-11-08T11:18:24,441 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915., pid=32, masterSystemTime=1731064704413 2024-11-08T11:18:24,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,444 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:18:24,444 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=0aa76efa23785cc8435444278f71e915, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:24,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T11:18:24,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 0aa76efa23785cc8435444278f71e915, server=ae52d81aa32e,33717,1731064629394 in 186 msec 2024-11-08T11:18:24,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T11:18:24,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=0aa76efa23785cc8435444278f71e915, ASSIGN in 346 msec 2024-11-08T11:18:24,452 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:24,453 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064704452"}]},"ts":"1731064704452"} 2024-11-08T11:18:24,454 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:24,457 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:24,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 808 msec 2024-11-08T11:18:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T11:18:24,658 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-08T11:18:24,658 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:24,658 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:24,667 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:24,667 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:24,667 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-08T11:18:24,672 DEBUG [hconnection-0x4152c0b1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:24,690 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54076, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:24,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:24,713 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:18:24,719 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300088 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:18:24,721 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T11:18:29,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T11:18:29,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:29,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T11:18:29,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:29,923 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:29,968 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-08T11:18:31,222 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6509 2024-11-08T11:18:31,222 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4feba97d to 127.0.0.1:55518 2024-11-08T11:18:31,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:31,223 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:31,223 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:31,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:31,223 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1a1dbdc4 to 127.0.0.1:55518 2024-11-08T11:18:31,236 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 228), OpenFileDescriptor=454 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 339) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2691 (was 2714) 2024-11-08T11:18:31,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=2690 2024-11-08T11:18:31,248 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x299eb4a0 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@73e13866 2024-11-08T11:18:31,256 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x299eb4a0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a17284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:31,258 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a2301ef to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@655b74f3 2024-11-08T11:18:31,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393cf74c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:31,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-08T11:18:31,267 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:31,268 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:31,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-08T11:18:31,269 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T11:18:31,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741853_1029 (size=421) 2024-11-08T11:18:31,680 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 47efa016b96d371d1d5a39cfe0614766, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:31,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741854_1030 (size=72) 2024-11-08T11:18:32,091 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:32,091 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 47efa016b96d371d1d5a39cfe0614766, disabling compactions & flushes 2024-11-08T11:18:32,091 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,091 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,091 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. after waiting 0 ms 2024-11-08T11:18:32,091 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,091 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,092 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 47efa016b96d371d1d5a39cfe0614766: 2024-11-08T11:18:32,094 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:32,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731064712094"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064712094"}]},"ts":"1731064712094"} 2024-11-08T11:18:32,097 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:32,098 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:32,098 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064712098"}]},"ts":"1731064712098"} 2024-11-08T11:18:32,100 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:32,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=47efa016b96d371d1d5a39cfe0614766, ASSIGN}] 2024-11-08T11:18:32,106 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=47efa016b96d371d1d5a39cfe0614766, ASSIGN 2024-11-08T11:18:32,107 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=47efa016b96d371d1d5a39cfe0614766, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:32,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=47efa016b96d371d1d5a39cfe0614766, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:32,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 47efa016b96d371d1d5a39cfe0614766, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T11:18:32,415 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:32,422 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,422 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => 47efa016b96d371d1d5a39cfe0614766, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:32,423 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,423 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:32,423 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for 47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,423 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for 47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,425 INFO [StoreOpener-47efa016b96d371d1d5a39cfe0614766-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,427 INFO [StoreOpener-47efa016b96d371d1d5a39cfe0614766-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47efa016b96d371d1d5a39cfe0614766 columnFamilyName testFamily 2024-11-08T11:18:32,427 DEBUG [StoreOpener-47efa016b96d371d1d5a39cfe0614766-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:32,427 INFO [StoreOpener-47efa016b96d371d1d5a39cfe0614766-1 {}] regionserver.HStore(327): Store=47efa016b96d371d1d5a39cfe0614766/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:32,429 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,430 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,433 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for 47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:18:32,436 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:32,437 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened 47efa016b96d371d1d5a39cfe0614766; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61942728, jitterRate=-0.07698142528533936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:32,439 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for 47efa016b96d371d1d5a39cfe0614766: 2024-11-08T11:18:32,440 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766., pid=35, masterSystemTime=1731064712415 2024-11-08T11:18:32,443 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,443 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:18:32,444 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=47efa016b96d371d1d5a39cfe0614766, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:32,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-08T11:18:32,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 47efa016b96d371d1d5a39cfe0614766, server=ae52d81aa32e,33717,1731064629394 in 184 msec 2024-11-08T11:18:32,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T11:18:32,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=47efa016b96d371d1d5a39cfe0614766, ASSIGN in 345 msec 2024-11-08T11:18:32,454 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:32,454 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064712454"}]},"ts":"1731064712454"} 2024-11-08T11:18:32,456 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:32,459 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:32,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 1.1950 sec 2024-11-08T11:18:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T11:18:34,275 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-08T11:18:34,275 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:34,275 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:34,281 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:34,281 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:34,281 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-08T11:18:34,284 DEBUG [hconnection-0x11e081e3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:34,286 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53254, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:34,301 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:34,311 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:18:34,317 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300092 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:18:34,317 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T11:18:35,818 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1506 2024-11-08T11:18:35,819 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a2301ef to 127.0.0.1:55518 2024-11-08T11:18:35,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:35,819 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:35,819 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:35,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:35,819 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x299eb4a0 to 127.0.0.1:55518 2024-11-08T11:18:35,828 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=2675 (was 2690) 2024-11-08T11:18:35,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=2675 2024-11-08T11:18:35,841 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c771504 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68c52a5f 2024-11-08T11:18:35,844 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x3c771504 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c8aa76f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:35,846 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x72f683b9 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f826340 2024-11-08T11:18:35,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b598a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:35,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-08T11:18:35,854 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:35,854 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:35,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-08T11:18:35,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T11:18:35,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741855_1031 (size=420) 2024-11-08T11:18:35,867 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 28a413aad77fe00c80109f757f9eff1d, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:35,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741856_1032 (size=71) 2024-11-08T11:18:35,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-08T11:18:36,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:36,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 28a413aad77fe00c80109f757f9eff1d, disabling compactions & flushes 2024-11-08T11:18:36,275 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. after waiting 0 ms 2024-11-08T11:18:36,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,276 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,276 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 28a413aad77fe00c80109f757f9eff1d: 2024-11-08T11:18:36,277 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:36,278 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731064716277"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064716277"}]},"ts":"1731064716277"} 2024-11-08T11:18:36,280 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:36,281 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:36,282 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064716281"}]},"ts":"1731064716281"} 2024-11-08T11:18:36,283 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:36,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=28a413aad77fe00c80109f757f9eff1d, ASSIGN}] 2024-11-08T11:18:36,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=28a413aad77fe00c80109f757f9eff1d, ASSIGN 2024-11-08T11:18:36,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=28a413aad77fe00c80109f757f9eff1d, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:36,444 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=28a413aad77fe00c80109f757f9eff1d, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:36,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 28a413aad77fe00c80109f757f9eff1d, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:36,599 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:36,604 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,604 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 28a413aad77fe00c80109f757f9eff1d, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:36,605 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,605 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:36,605 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,605 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,607 INFO [StoreOpener-28a413aad77fe00c80109f757f9eff1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,609 INFO [StoreOpener-28a413aad77fe00c80109f757f9eff1d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a413aad77fe00c80109f757f9eff1d columnFamilyName testFamily 2024-11-08T11:18:36,609 DEBUG [StoreOpener-28a413aad77fe00c80109f757f9eff1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:36,609 INFO [StoreOpener-28a413aad77fe00c80109f757f9eff1d-1 {}] regionserver.HStore(327): Store=28a413aad77fe00c80109f757f9eff1d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:36,610 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,611 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,613 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:18:36,616 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:36,617 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 28a413aad77fe00c80109f757f9eff1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64080947, jitterRate=-0.045119479298591614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:36,617 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 28a413aad77fe00c80109f757f9eff1d: 2024-11-08T11:18:36,618 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d., pid=38, masterSystemTime=1731064716599 2024-11-08T11:18:36,620 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,621 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:18:36,621 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=28a413aad77fe00c80109f757f9eff1d, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:36,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-08T11:18:36,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 28a413aad77fe00c80109f757f9eff1d, server=ae52d81aa32e,33717,1731064629394 in 177 msec 2024-11-08T11:18:36,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T11:18:36,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=28a413aad77fe00c80109f757f9eff1d, ASSIGN in 334 msec 2024-11-08T11:18:36,628 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:36,628 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064716628"}]},"ts":"1731064716628"} 2024-11-08T11:18:36,630 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:36,632 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:36,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 781 msec 2024-11-08T11:18:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T11:18:36,858 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-08T11:18:36,858 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:36,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:36,863 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:36,863 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:36,863 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-08T11:18:36,865 DEBUG [hconnection-0x476c1276-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:36,867 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:36,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:36,884 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7abf1d24 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4649b214 2024-11-08T11:18:36,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f2458c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:36,893 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:18:36,895 DEBUG [hconnection-0x3ccf54f2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:36,904 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53266, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:36,908 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:18:36,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300097 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:36,909 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:18:36,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300097 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:18:36,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T11:18:37,771 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:18:37,917 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T11:18:37.917Z, RpcRetryingCaller{globalStartTime=2024-11-08T11:18:36.910Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T11:18:37.917Z, RpcRetryingCaller{globalStartTime=2024-11-08T11:18:36.910Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:18:37,918 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1008 2024-11-08T11:18:37,918 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7abf1d24 to 127.0.0.1:55518 2024-11-08T11:18:37,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:37,919 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x72f683b9 to 127.0.0.1:55518 2024-11-08T11:18:37,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:37,919 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:37,919 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:37,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:37,919 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c771504 to 127.0.0.1:55518 2024-11-08T11:18:37,925 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:37,930 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=235 (was 231) - Thread LEAK? -, OpenFileDescriptor=459 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=2638 (was 2675) 2024-11-08T11:18:37,941 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=2638 2024-11-08T11:18:37,943 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x79fb0e07 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@660384b 2024-11-08T11:18:37,946 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x79fb0e07 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6141fff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:37,947 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45eb3de1 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53f60566 2024-11-08T11:18:37,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72dcb229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:37,954 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:37,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T11:18:37,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:37,956 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:37,956 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-08T11:18:37,957 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T11:18:37,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741857_1033 (size=422) 2024-11-08T11:18:38,370 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8854a0dae2dc4ae8fc413fa10c0ab93c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:38,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741858_1034 (size=73) 2024-11-08T11:18:38,411 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:18:38,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:38,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 8854a0dae2dc4ae8fc413fa10c0ab93c, disabling compactions & flushes 2024-11-08T11:18:38,779 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:38,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:38,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. after waiting 0 ms 2024-11-08T11:18:38,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:38,779 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:38,779 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8854a0dae2dc4ae8fc413fa10c0ab93c: 2024-11-08T11:18:38,780 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:38,781 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731064718781"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064718781"}]},"ts":"1731064718781"} 2024-11-08T11:18:38,783 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:38,784 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:38,784 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064718784"}]},"ts":"1731064718784"} 2024-11-08T11:18:38,785 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:38,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=8854a0dae2dc4ae8fc413fa10c0ab93c, ASSIGN}] 2024-11-08T11:18:38,791 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=8854a0dae2dc4ae8fc413fa10c0ab93c, ASSIGN 2024-11-08T11:18:38,791 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=8854a0dae2dc4ae8fc413fa10c0ab93c, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:38,942 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=8854a0dae2dc4ae8fc413fa10c0ab93c, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:38,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 8854a0dae2dc4ae8fc413fa10c0ab93c, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T11:18:39,097 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:39,101 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:39,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 8854a0dae2dc4ae8fc413fa10c0ab93c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:39,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:39,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,104 INFO [StoreOpener-8854a0dae2dc4ae8fc413fa10c0ab93c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,106 INFO [StoreOpener-8854a0dae2dc4ae8fc413fa10c0ab93c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8854a0dae2dc4ae8fc413fa10c0ab93c columnFamilyName testFamily 2024-11-08T11:18:39,106 DEBUG [StoreOpener-8854a0dae2dc4ae8fc413fa10c0ab93c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:39,107 INFO [StoreOpener-8854a0dae2dc4ae8fc413fa10c0ab93c-1 {}] regionserver.HStore(327): Store=8854a0dae2dc4ae8fc413fa10c0ab93c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:39,107 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,108 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,110 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:18:39,113 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:39,114 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 8854a0dae2dc4ae8fc413fa10c0ab93c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68504890, jitterRate=0.020802408456802368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:39,114 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 8854a0dae2dc4ae8fc413fa10c0ab93c: 2024-11-08T11:18:39,115 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c., pid=41, masterSystemTime=1731064719097 2024-11-08T11:18:39,117 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:39,117 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:18:39,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=8854a0dae2dc4ae8fc413fa10c0ab93c, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:39,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T11:18:39,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 8854a0dae2dc4ae8fc413fa10c0ab93c, server=ae52d81aa32e,33717,1731064629394 in 176 msec 2024-11-08T11:18:39,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T11:18:39,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:39,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-08T11:18:39,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=8854a0dae2dc4ae8fc413fa10c0ab93c, ASSIGN in 333 msec 2024-11-08T11:18:39,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T11:18:39,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:39,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:39,125 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064719125"}]},"ts":"1731064719125"} 2024-11-08T11:18:39,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T11:18:39,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:18:39,126 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:39,129 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:39,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.1750 sec 2024-11-08T11:18:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T11:18:40,961 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-08T11:18:40,962 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:40,962 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:40,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:40,966 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:40,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T11:18:40,968 DEBUG [hconnection-0x1d8512d5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:40,969 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37282, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:40,978 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:40,980 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:18:40,982 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:18:40,982 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T11:18:41,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-08T11:18:41,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-08T11:18:42,793 DEBUG [master/ae52d81aa32e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T11:18:42,797 DEBUG [master/ae52d81aa32e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region ad75048110dcede8a9d5403d1e5644c9 changed from -1.0 to 0.0, refreshing cache 2024-11-08T11:18:44,604 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:45,455 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:18:45,853 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:18:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T11:18:47,995 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:18:47,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37282 deadline: 1731064729982, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:18:47,996 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:18:47,997 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300101 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:18:47,997 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7017 2024-11-08T11:18:47,998 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x45eb3de1 to 127.0.0.1:55518 2024-11-08T11:18:47,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:47,998 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:18:47,998 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:18:47,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:18:47,998 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x79fb0e07 to 127.0.0.1:55518 2024-11-08T11:18:48,007 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=229 (was 236), OpenFileDescriptor=450 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=2655 (was 2638) - AvailableMemoryMB LEAK? - 2024-11-08T11:18:48,015 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=2654 2024-11-08T11:18:48,017 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17a007ae to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1d771c6e 2024-11-08T11:18:48,020 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x17a007ae {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1015c425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:48,021 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1345e320 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48186989 2024-11-08T11:18:48,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f928a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:18:48,028 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:18:48,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T11:18:48,030 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:18:48,030 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:48,030 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-08T11:18:48,031 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:18:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T11:18:48,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741859_1035 (size=428) 2024-11-08T11:18:48,441 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8029e11915c8ac0360de37aa6659111e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:18:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741860_1036 (size=79) 2024-11-08T11:18:48,849 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:48,849 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 8029e11915c8ac0360de37aa6659111e, disabling compactions & flushes 2024-11-08T11:18:48,849 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:48,849 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:48,849 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. after waiting 0 ms 2024-11-08T11:18:48,849 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:48,849 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:48,849 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8029e11915c8ac0360de37aa6659111e: 2024-11-08T11:18:48,851 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:18:48,851 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731064728851"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064728851"}]},"ts":"1731064728851"} 2024-11-08T11:18:48,853 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:18:48,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:18:48,854 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064728854"}]},"ts":"1731064728854"} 2024-11-08T11:18:48,856 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:18:48,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8029e11915c8ac0360de37aa6659111e, ASSIGN}] 2024-11-08T11:18:48,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8029e11915c8ac0360de37aa6659111e, ASSIGN 2024-11-08T11:18:48,862 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8029e11915c8ac0360de37aa6659111e, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:18:49,012 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=8029e11915c8ac0360de37aa6659111e, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:49,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 8029e11915c8ac0360de37aa6659111e, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:18:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T11:18:49,166 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:49,172 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:49,172 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 8029e11915c8ac0360de37aa6659111e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:18:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:18:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,175 INFO [StoreOpener-8029e11915c8ac0360de37aa6659111e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,176 INFO [StoreOpener-8029e11915c8ac0360de37aa6659111e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8029e11915c8ac0360de37aa6659111e columnFamilyName testFamily 2024-11-08T11:18:49,177 DEBUG [StoreOpener-8029e11915c8ac0360de37aa6659111e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:18:49,177 INFO [StoreOpener-8029e11915c8ac0360de37aa6659111e-1 {}] regionserver.HStore(327): Store=8029e11915c8ac0360de37aa6659111e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:18:49,178 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,178 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,181 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 8029e11915c8ac0360de37aa6659111e 2024-11-08T11:18:49,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:18:49,184 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 8029e11915c8ac0360de37aa6659111e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59134212, jitterRate=-0.1188315749168396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:18:49,185 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 8029e11915c8ac0360de37aa6659111e: 2024-11-08T11:18:49,186 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., pid=44, masterSystemTime=1731064729166 2024-11-08T11:18:49,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:49,188 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:18:49,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=8029e11915c8ac0360de37aa6659111e, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:18:49,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-08T11:18:49,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 8029e11915c8ac0360de37aa6659111e, server=ae52d81aa32e,33717,1731064629394 in 177 msec 2024-11-08T11:18:49,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T11:18:49,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=8029e11915c8ac0360de37aa6659111e, ASSIGN in 333 msec 2024-11-08T11:18:49,195 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:18:49,195 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064729195"}]},"ts":"1731064729195"} 2024-11-08T11:18:49,197 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:18:49,200 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:18:49,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.1720 sec 2024-11-08T11:18:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T11:18:51,035 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-08T11:18:51,035 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:18:51,036 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:51,041 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T11:18:51,041 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:18:51,041 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T11:18:51,044 DEBUG [hconnection-0x3f4a1f88-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:51,045 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45888, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:51,055 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:18:51,058 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:18:51,058 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:18:51,060 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x17a007ae {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:18:51,060 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x17a007ae {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:18:51,061 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45890, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:18:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:18:51,063 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:18:53,974 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T11:18:54,675 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:18:54,689 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300052 expired clientIPAndPort=172.17.0.2:51842, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:18:57,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2024-11-08T11:18:57,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2024-11-08T11:18:57,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:18:57,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T11:18:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:18:58,079 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:18:59,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T11:18:59,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:00,067 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1988ms, rpcTimeout=1979ms 2024-11-08T11:19:00,067 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1988ms, rpcTimeout=1979ms 2024-11-08T11:19:00,067 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:19:00,067 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T11:18:57.069Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2024-11-08T11:19:00.067Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1988ms, rpcTimeout=1979ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1988ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1988ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T11:19:00,069 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9011 2024-11-08T11:19:00,070 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1345e320 to 127.0.0.1:55518 2024-11-08T11:19:00,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:00,070 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:00,070 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:00,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:00,070 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17a007ae to 127.0.0.1:55518 2024-11-08T11:19:00,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=228 (was 229), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 315) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2576 (was 2654) 2024-11-08T11:19:00,093 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=228, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=2576 2024-11-08T11:19:00,096 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4be60061 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a2f0601 2024-11-08T11:19:00,100 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4be60061 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6361ddf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:00,102 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b590599 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cf49d82 2024-11-08T11:19:00,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a6b75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:00,108 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:00,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-08T11:19:00,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:00,111 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:00,111 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-08T11:19:00,112 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T11:19:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741861_1037 (size=416) 2024-11-08T11:19:00,540 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7fdf3e5624b7e358e192563c0644e9a1, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:00,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741862_1038 (size=67) 2024-11-08T11:19:00,950 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:00,950 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 7fdf3e5624b7e358e192563c0644e9a1, disabling compactions & flushes 2024-11-08T11:19:00,951 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:00,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:00,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. after waiting 0 ms 2024-11-08T11:19:00,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:00,951 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:00,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7fdf3e5624b7e358e192563c0644e9a1: 2024-11-08T11:19:00,953 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:00,954 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731064740953"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064740953"}]},"ts":"1731064740953"} 2024-11-08T11:19:00,956 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:00,957 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:00,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064740957"}]},"ts":"1731064740957"} 2024-11-08T11:19:00,959 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:19:00,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=7fdf3e5624b7e358e192563c0644e9a1, ASSIGN}] 2024-11-08T11:19:00,969 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=7fdf3e5624b7e358e192563c0644e9a1, ASSIGN 2024-11-08T11:19:00,970 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=7fdf3e5624b7e358e192563c0644e9a1, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T11:19:01,120 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=7fdf3e5624b7e358e192563c0644e9a1, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:01,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 7fdf3e5624b7e358e192563c0644e9a1, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:01,275 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:01,291 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:01,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 7fdf3e5624b7e358e192563c0644e9a1, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:01,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:01,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,295 INFO [StoreOpener-7fdf3e5624b7e358e192563c0644e9a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,298 INFO [StoreOpener-7fdf3e5624b7e358e192563c0644e9a1-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdf3e5624b7e358e192563c0644e9a1 columnFamilyName testFamily 2024-11-08T11:19:01,298 DEBUG [StoreOpener-7fdf3e5624b7e358e192563c0644e9a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:01,298 INFO [StoreOpener-7fdf3e5624b7e358e192563c0644e9a1-1 {}] regionserver.HStore(327): Store=7fdf3e5624b7e358e192563c0644e9a1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:01,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:19:01,311 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:01,312 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 7fdf3e5624b7e358e192563c0644e9a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69326004, jitterRate=0.03303796052932739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:01,314 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 7fdf3e5624b7e358e192563c0644e9a1: 2024-11-08T11:19:01,317 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1., pid=47, masterSystemTime=1731064741275 2024-11-08T11:19:01,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:01,320 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:19:01,323 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=7fdf3e5624b7e358e192563c0644e9a1, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:01,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T11:19:01,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 7fdf3e5624b7e358e192563c0644e9a1, server=ae52d81aa32e,33717,1731064629394 in 203 msec 2024-11-08T11:19:01,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-08T11:19:01,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=7fdf3e5624b7e358e192563c0644e9a1, ASSIGN in 362 msec 2024-11-08T11:19:01,332 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:01,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064741332"}]},"ts":"1731064741332"} 2024-11-08T11:19:01,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:19:01,338 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:01,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.2300 sec 2024-11-08T11:19:01,695 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300054 expired clientIPAndPort=172.17.0.2:51842, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:19:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T11:19:03,118 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-08T11:19:03,118 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:19:03,118 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:03,124 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T11:19:03,124 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:03,124 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-08T11:19:03,127 DEBUG [hconnection-0x5a54b6c4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:03,128 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36576, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:03,138 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:03,145 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d6307c7 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d9863e3 2024-11-08T11:19:03,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1a86be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:03,149 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:19:03,156 DEBUG [hconnection-0x25737345-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:03,158 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36592, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:03,162 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:19:03,163 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300114 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:03,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:19:03,168 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300114 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:03,169 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T11:19:04,580 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:05,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-08T11:19:06,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:19:07,771 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:19:07,805 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300059 expired clientIPAndPort=172.17.0.2:47878, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:19:09,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T11:19:09,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:09,177 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T11:19:09.177Z, RpcRetryingCaller{globalStartTime=2024-11-08T11:19:03.168Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T11:19:09.177Z, RpcRetryingCaller{globalStartTime=2024-11-08T11:19:03.168Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:19:09,178 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6013 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d6307c7 to 127.0.0.1:55518 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b590599 to 127.0.0.1:55518 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:09,178 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:09,178 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4be60061 to 127.0.0.1:55518 2024-11-08T11:19:09,191 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=234 (was 228) - Thread LEAK? -, OpenFileDescriptor=451 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 359) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2534 (was 2576) 2024-11-08T11:19:09,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=233, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=2542 2024-11-08T11:19:09,206 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a20bde7 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2bbee81d 2024-11-08T11:19:09,216 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x0a20bde7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553ec143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:09,217 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40534377 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6152a06b 2024-11-08T11:19:09,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e92f623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:09,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T11:19:09,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:09,235 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:09,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-08T11:19:09,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T11:19:09,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741863_1039 (size=432) 2024-11-08T11:19:09,669 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:09,681 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ce6c8c461db85938ff504cfbc5c00973, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:09,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741864_1040 (size=83) 2024-11-08T11:19:10,089 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:10,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ce6c8c461db85938ff504cfbc5c00973, disabling compactions & flushes 2024-11-08T11:19:10,090 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. after waiting 0 ms 2024-11-08T11:19:10,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,090 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,090 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ce6c8c461db85938ff504cfbc5c00973: 2024-11-08T11:19:10,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:10,091 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731064750091"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064750091"}]},"ts":"1731064750091"} 2024-11-08T11:19:10,093 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:10,094 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:10,094 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064750094"}]},"ts":"1731064750094"} 2024-11-08T11:19:10,096 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T11:19:10,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ce6c8c461db85938ff504cfbc5c00973, ASSIGN}] 2024-11-08T11:19:10,100 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ce6c8c461db85938ff504cfbc5c00973, ASSIGN 2024-11-08T11:19:10,101 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ce6c8c461db85938ff504cfbc5c00973, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T11:19:10,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=ce6c8c461db85938ff504cfbc5c00973, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:10,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure ce6c8c461db85938ff504cfbc5c00973, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:10,308 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300063 expired clientIPAndPort=172.17.0.2:46542, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:19:10,405 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:10,408 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300066 expired clientIPAndPort=172.17.0.2:46542, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:19:10,408 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300065 expired clientIPAndPort=172.17.0.2:46542, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:19:10,410 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,410 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => ce6c8c461db85938ff504cfbc5c00973, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:10,411 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,411 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:10,411 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,411 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,413 INFO [StoreOpener-ce6c8c461db85938ff504cfbc5c00973-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,414 INFO [StoreOpener-ce6c8c461db85938ff504cfbc5c00973-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce6c8c461db85938ff504cfbc5c00973 columnFamilyName testFamily 2024-11-08T11:19:10,414 DEBUG [StoreOpener-ce6c8c461db85938ff504cfbc5c00973-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:10,415 INFO [StoreOpener-ce6c8c461db85938ff504cfbc5c00973-1 {}] regionserver.HStore(327): Store=ce6c8c461db85938ff504cfbc5c00973/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:10,416 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,416 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,418 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:19:10,421 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:10,422 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened ce6c8c461db85938ff504cfbc5c00973; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74887399, jitterRate=0.11590920388698578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:10,423 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for ce6c8c461db85938ff504cfbc5c00973: 2024-11-08T11:19:10,424 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., pid=50, masterSystemTime=1731064750405 2024-11-08T11:19:10,426 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,426 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:19:10,426 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=ce6c8c461db85938ff504cfbc5c00973, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:10,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-08T11:19:10,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure ce6c8c461db85938ff504cfbc5c00973, server=ae52d81aa32e,33717,1731064629394 in 175 msec 2024-11-08T11:19:10,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T11:19:10,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ce6c8c461db85938ff504cfbc5c00973, ASSIGN in 331 msec 2024-11-08T11:19:10,432 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:10,432 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064750432"}]},"ts":"1731064750432"} 2024-11-08T11:19:10,434 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T11:19:10,438 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:10,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.2060 sec 2024-11-08T11:19:11,978 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T11:19:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T11:19:12,242 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-08T11:19:12,242 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T11:19:12,242 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:12,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T11:19:12,247 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:12,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T11:19:12,249 DEBUG [hconnection-0x655dcaec-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:12,251 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54490, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:12,260 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:12,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:19:12,263 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:19:12,265 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x0a20bde7 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:19:12,265 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x0a20bde7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:12,266 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54502, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:19:12,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:19:13,277 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-08T11:19:13,277 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-08T11:19:13,277 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:19:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T11:19:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:19:14,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:19:15,266 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=969ms, rpcTimeout=966ms 2024-11-08T11:19:15,267 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=969ms, rpcTimeout=966ms 2024-11-08T11:19:15,267 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:19:15,267 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T11:19:13.277Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-08T11:19:15.267Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=969ms, rpcTimeout=966ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=969ms, rpcTimeout=966ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=969ms, rpcTimeout=966ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T11:19:15,268 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2024-11-08T11:19:15,268 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x40534377 to 127.0.0.1:55518 2024-11-08T11:19:15,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:15,268 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:15,268 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:15,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:15,269 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a20bde7 to 127.0.0.1:55518 2024-11-08T11:19:15,278 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 233), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 380), ProcessCount=11 (was 11), AvailableMemoryMB=2534 (was 2542) 2024-11-08T11:19:15,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=2533 2024-11-08T11:19:15,288 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f446b87 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e56dc8c 2024-11-08T11:19:15,291 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4f446b87 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0dcc53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:15,291 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x120bb3af to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5de30fcb 2024-11-08T11:19:15,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31975fa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:15,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T11:19:15,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:15,299 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:15,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-08T11:19:15,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T11:19:15,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741865_1041 (size=436) 2024-11-08T11:19:15,412 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300072 expired clientIPAndPort=172.17.0.2:46544, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:19:15,710 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b96aee56c445cd36471b3c7201edc90d, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:15,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741866_1042 (size=87) 2024-11-08T11:19:15,798 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:15,912 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:19:16,118 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:16,118 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing b96aee56c445cd36471b3c7201edc90d, disabling compactions & flushes 2024-11-08T11:19:16,118 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,118 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,118 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. after waiting 0 ms 2024-11-08T11:19:16,118 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,118 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,118 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for b96aee56c445cd36471b3c7201edc90d: 2024-11-08T11:19:16,120 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:16,120 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731064756120"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064756120"}]},"ts":"1731064756120"} 2024-11-08T11:19:16,122 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:16,123 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:16,123 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064756123"}]},"ts":"1731064756123"} 2024-11-08T11:19:16,124 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:16,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=b96aee56c445cd36471b3c7201edc90d, ASSIGN}] 2024-11-08T11:19:16,131 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=b96aee56c445cd36471b3c7201edc90d, ASSIGN 2024-11-08T11:19:16,132 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=b96aee56c445cd36471b3c7201edc90d, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:16,282 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=b96aee56c445cd36471b3c7201edc90d, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:16,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure b96aee56c445cd36471b3c7201edc90d, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T11:19:16,436 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:16,441 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,441 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => b96aee56c445cd36471b3c7201edc90d, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:16,442 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,442 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:16,442 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,442 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,444 INFO [StoreOpener-b96aee56c445cd36471b3c7201edc90d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,446 INFO [StoreOpener-b96aee56c445cd36471b3c7201edc90d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b96aee56c445cd36471b3c7201edc90d columnFamilyName testFamily 2024-11-08T11:19:16,446 DEBUG [StoreOpener-b96aee56c445cd36471b3c7201edc90d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:16,446 INFO [StoreOpener-b96aee56c445cd36471b3c7201edc90d-1 {}] regionserver.HStore(327): Store=b96aee56c445cd36471b3c7201edc90d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:16,447 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,448 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,450 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:19:16,452 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:16,452 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened b96aee56c445cd36471b3c7201edc90d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65894068, jitterRate=-0.018101871013641357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:16,453 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for b96aee56c445cd36471b3c7201edc90d: 2024-11-08T11:19:16,454 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d., pid=53, masterSystemTime=1731064756436 2024-11-08T11:19:16,456 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,456 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:19:16,457 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=b96aee56c445cd36471b3c7201edc90d, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:16,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T11:19:16,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure b96aee56c445cd36471b3c7201edc90d, server=ae52d81aa32e,33717,1731064629394 in 173 msec 2024-11-08T11:19:16,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-08T11:19:16,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=b96aee56c445cd36471b3c7201edc90d, ASSIGN in 330 msec 2024-11-08T11:19:16,462 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:16,462 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064756462"}]},"ts":"1731064756462"} 2024-11-08T11:19:16,464 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:16,472 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:16,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1750 sec 2024-11-08T11:19:17,979 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-08T11:19:18,014 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300077 expired clientIPAndPort=172.17.0.2:46556, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:19:18,036 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:19:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T11:19:18,303 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-08T11:19:18,303 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:18,304 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:18,308 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:18,308 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:18,308 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-08T11:19:18,311 DEBUG [hconnection-0x38cde223-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:18,312 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54518, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:18,319 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:18,321 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T11:19:18,321 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:19:18,322 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4f446b87 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:19:18,323 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4f446b87 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:18,324 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54528, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:19:18,326 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T11:19:18,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300126 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:18,327 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T11:19:18,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300126 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:18,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300126 2024-11-08T11:19:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:54528 deadline: 1731064761326, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:18,328 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300126 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:18,329 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T11:19:18,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300127 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:18,330 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T11:19:18,330 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:18,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:18,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300128 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:18,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300128 2024-11-08T11:19:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:54528 deadline: 1731064761330, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:18,332 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300128 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:18,333 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:18,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300129 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:18,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4132839332912300129 2024-11-08T11:19:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:54528 deadline: 1731064761332, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:18,334 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300129 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:18,335 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:18,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300130 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:18,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4132839332912300130 2024-11-08T11:19:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:54528 deadline: 1731064761334, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:18,336 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300130 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:18,337 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:18,337 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x120bb3af to 127.0.0.1:55518 2024-11-08T11:19:18,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:18,337 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:18,337 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:18,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:18,338 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f446b87 to 127.0.0.1:55518 2024-11-08T11:19:18,347 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 358), ProcessCount=11 (was 11), AvailableMemoryMB=2523 (was 2533) 2024-11-08T11:19:18,355 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=232, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=329, ProcessCount=11, AvailableMemoryMB=2522 2024-11-08T11:19:18,357 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5993a56a to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24dbfdae 2024-11-08T11:19:18,360 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x5993a56a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30228f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:18,361 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1683a517 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@43a4445b 2024-11-08T11:19:18,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d801b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:18,367 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:18,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T11:19:18,368 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:18,369 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:18,369 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-08T11:19:18,369 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T11:19:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741867_1043 (size=422) 2024-11-08T11:19:18,779 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b6492f1f8d3b65f481a71f69b3f63ffa, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:18,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741868_1044 (size=73) 2024-11-08T11:19:19,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T11:19:19,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:19,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T11:19:19,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:19,186 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:19:19,187 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:19,187 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing b6492f1f8d3b65f481a71f69b3f63ffa, disabling compactions & flushes 2024-11-08T11:19:19,187 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,187 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,187 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. after waiting 0 ms 2024-11-08T11:19:19,187 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,187 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,187 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for b6492f1f8d3b65f481a71f69b3f63ffa: 2024-11-08T11:19:19,188 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:19,189 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731064759189"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064759189"}]},"ts":"1731064759189"} 2024-11-08T11:19:19,191 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:19,191 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:19,191 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064759191"}]},"ts":"1731064759191"} 2024-11-08T11:19:19,193 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:19,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=b6492f1f8d3b65f481a71f69b3f63ffa, ASSIGN}] 2024-11-08T11:19:19,197 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=b6492f1f8d3b65f481a71f69b3f63ffa, ASSIGN 2024-11-08T11:19:19,198 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=b6492f1f8d3b65f481a71f69b3f63ffa, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:19,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=b6492f1f8d3b65f481a71f69b3f63ffa, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:19,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure b6492f1f8d3b65f481a71f69b3f63ffa, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T11:19:19,502 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:19,506 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,507 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => b6492f1f8d3b65f481a71f69b3f63ffa, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:19,507 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,507 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:19,507 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,507 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,509 INFO [StoreOpener-b6492f1f8d3b65f481a71f69b3f63ffa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,511 INFO [StoreOpener-b6492f1f8d3b65f481a71f69b3f63ffa-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6492f1f8d3b65f481a71f69b3f63ffa columnFamilyName testFamily 2024-11-08T11:19:19,511 DEBUG [StoreOpener-b6492f1f8d3b65f481a71f69b3f63ffa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:19,511 INFO [StoreOpener-b6492f1f8d3b65f481a71f69b3f63ffa-1 {}] regionserver.HStore(327): Store=b6492f1f8d3b65f481a71f69b3f63ffa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:19,513 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,513 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,516 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:19:19,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:19,519 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened b6492f1f8d3b65f481a71f69b3f63ffa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59382717, jitterRate=-0.11512856185436249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:19,520 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for b6492f1f8d3b65f481a71f69b3f63ffa: 2024-11-08T11:19:19,521 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa., pid=56, masterSystemTime=1731064759502 2024-11-08T11:19:19,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,523 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:19:19,524 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=b6492f1f8d3b65f481a71f69b3f63ffa, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:19,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T11:19:19,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure b6492f1f8d3b65f481a71f69b3f63ffa, server=ae52d81aa32e,33717,1731064629394 in 175 msec 2024-11-08T11:19:19,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T11:19:19,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=b6492f1f8d3b65f481a71f69b3f63ffa, ASSIGN in 331 msec 2024-11-08T11:19:19,529 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:19,529 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064759529"}]},"ts":"1731064759529"} 2024-11-08T11:19:19,530 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:19,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:19,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1660 sec 2024-11-08T11:19:20,016 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300079 expired clientIPAndPort=172.17.0.2:46556, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:19:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T11:19:21,374 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-08T11:19:21,374 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:21,374 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:21,377 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:21,377 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:21,378 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T11:19:21,380 DEBUG [hconnection-0x51c0413a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:21,381 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38106, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:21,388 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:21,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:19:21,391 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:19:21,943 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:19:23,980 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-08T11:19:24,120 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300084 expired clientIPAndPort=172.17.0.2:54068, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:19:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T11:19:28,424 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:19:29,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T11:19:29,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:30,397 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11018: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11018: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:19:30,398 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T11:19:30.398Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11018: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11018: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1974ms, rpcTimeout=1967ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:19:30,399 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-08T11:19:30,399 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1683a517 to 127.0.0.1:55518 2024-11-08T11:19:30,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:30,400 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:30,400 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:30,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:30,400 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5993a56a to 127.0.0.1:55518 2024-11-08T11:19:30,411 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=228 (was 232), OpenFileDescriptor=446 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=302 (was 329), ProcessCount=11 (was 11), AvailableMemoryMB=2498 (was 2522) 2024-11-08T11:19:30,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=228, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=302, ProcessCount=11, AvailableMemoryMB=2498 2024-11-08T11:19:30,425 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4549c99b to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33ddf85a 2024-11-08T11:19:30,440 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4549c99b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184e3b18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:30,441 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3318ab91 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b7de2b9 2024-11-08T11:19:30,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f3e95c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:30,454 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:30,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T11:19:30,456 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:30,456 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:30,456 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-08T11:19:30,457 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T11:19:30,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741869_1045 (size=420) 2024-11-08T11:19:30,881 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 012d9e50c5a5dfe248e1e5d5c6aa397b, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:30,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741870_1046 (size=71) 2024-11-08T11:19:30,901 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:30,901 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 012d9e50c5a5dfe248e1e5d5c6aa397b, disabling compactions & flushes 2024-11-08T11:19:30,901 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:30,902 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:30,902 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. after waiting 0 ms 2024-11-08T11:19:30,902 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:30,902 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:30,902 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 012d9e50c5a5dfe248e1e5d5c6aa397b: 2024-11-08T11:19:30,903 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:30,904 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731064770903"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064770903"}]},"ts":"1731064770903"} 2024-11-08T11:19:30,906 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:30,907 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:30,907 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064770907"}]},"ts":"1731064770907"} 2024-11-08T11:19:30,909 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:30,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=012d9e50c5a5dfe248e1e5d5c6aa397b, ASSIGN}] 2024-11-08T11:19:30,915 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=012d9e50c5a5dfe248e1e5d5c6aa397b, ASSIGN 2024-11-08T11:19:30,916 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=012d9e50c5a5dfe248e1e5d5c6aa397b, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:31,067 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=012d9e50c5a5dfe248e1e5d5c6aa397b, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:31,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 012d9e50c5a5dfe248e1e5d5c6aa397b, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:31,222 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:31,229 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:31,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => 012d9e50c5a5dfe248e1e5d5c6aa397b, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:31,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:31,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for 012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for 012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,232 INFO [StoreOpener-012d9e50c5a5dfe248e1e5d5c6aa397b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,234 INFO [StoreOpener-012d9e50c5a5dfe248e1e5d5c6aa397b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 012d9e50c5a5dfe248e1e5d5c6aa397b columnFamilyName testFamily 2024-11-08T11:19:31,234 DEBUG [StoreOpener-012d9e50c5a5dfe248e1e5d5c6aa397b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:31,234 INFO [StoreOpener-012d9e50c5a5dfe248e1e5d5c6aa397b-1 {}] regionserver.HStore(327): Store=012d9e50c5a5dfe248e1e5d5c6aa397b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:31,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,239 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for 012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:19:31,243 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:31,244 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened 012d9e50c5a5dfe248e1e5d5c6aa397b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58947781, jitterRate=-0.12160961329936981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:31,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for 012d9e50c5a5dfe248e1e5d5c6aa397b: 2024-11-08T11:19:31,246 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b., pid=59, masterSystemTime=1731064771221 2024-11-08T11:19:31,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:31,248 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:19:31,249 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=012d9e50c5a5dfe248e1e5d5c6aa397b, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:31,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T11:19:31,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 012d9e50c5a5dfe248e1e5d5c6aa397b, server=ae52d81aa32e,33717,1731064629394 in 183 msec 2024-11-08T11:19:31,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T11:19:31,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=012d9e50c5a5dfe248e1e5d5c6aa397b, ASSIGN in 340 msec 2024-11-08T11:19:31,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:31,257 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064771257"}]},"ts":"1731064771257"} 2024-11-08T11:19:31,258 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:31,262 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:31,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 808 msec 2024-11-08T11:19:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T11:19:31,460 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-08T11:19:31,460 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:31,460 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:31,464 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:31,464 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:31,464 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T11:19:31,467 DEBUG [hconnection-0x21a8425d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:31,468 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38850, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:31,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:31,479 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:19:31,479 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:19:31,480 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4549c99b {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:19:31,481 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4549c99b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:31,482 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38858, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:19:31,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:19:31,484 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300142 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:31,485 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:19:31,485 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300142 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:31,485 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T11:19:34,925 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:35,982 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-08T11:19:36,792 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:19:37,771 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:19:38,432 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300097 expired clientIPAndPort=172.17.0.2:53266, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:19:39,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T11:19:39,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:40,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9002 2024-11-08T11:19:40,488 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3318ab91 to 127.0.0.1:55518 2024-11-08T11:19:40,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:40,488 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:40,488 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300142 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:40,488 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:40,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:40,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300142' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T11:19:40,488 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4549c99b to 127.0.0.1:55518 2024-11-08T11:19:40,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:38858 deadline: 1731064786487, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300142' does not exist 2024-11-08T11:19:40,489 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:40,488 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ae52d81aa32e,33717,1731064629394 for closing scanner id = 4132839332912300142 for 012d9e50c5a5dfe248e1e5d5c6aa397b of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T11:19:40,500 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=448 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 302) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2465 (was 2498) 2024-11-08T11:19:40,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=326, ProcessCount=11, AvailableMemoryMB=2465 2024-11-08T11:19:40,510 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3201081b to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@128b6141 2024-11-08T11:19:40,515 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x3201081b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19fb8a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:40,516 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1297d3a3 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f0a47df 2024-11-08T11:19:40,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f1c3f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:40,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T11:19:40,525 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:40,525 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:40,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-08T11:19:40,525 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T11:19:40,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741871_1047 (size=431) 2024-11-08T11:19:40,941 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 810cb95db16ea2b3418f12a23ced7304, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:40,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741872_1048 (size=82) 2024-11-08T11:19:40,986 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:41,351 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:41,351 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 810cb95db16ea2b3418f12a23ced7304, disabling compactions & flushes 2024-11-08T11:19:41,351 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,351 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,351 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. after waiting 0 ms 2024-11-08T11:19:41,351 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,351 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,351 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 810cb95db16ea2b3418f12a23ced7304: 2024-11-08T11:19:41,353 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:41,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731064781353"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064781353"}]},"ts":"1731064781353"} 2024-11-08T11:19:41,355 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:41,356 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:41,357 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064781356"}]},"ts":"1731064781356"} 2024-11-08T11:19:41,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:41,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=810cb95db16ea2b3418f12a23ced7304, ASSIGN}] 2024-11-08T11:19:41,364 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=810cb95db16ea2b3418f12a23ced7304, ASSIGN 2024-11-08T11:19:41,365 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=810cb95db16ea2b3418f12a23ced7304, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:41,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=810cb95db16ea2b3418f12a23ced7304, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:41,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 810cb95db16ea2b3418f12a23ced7304, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T11:19:41,670 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:41,676 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,676 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 810cb95db16ea2b3418f12a23ced7304, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:41,677 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,677 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:41,677 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,677 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,679 INFO [StoreOpener-810cb95db16ea2b3418f12a23ced7304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,681 INFO [StoreOpener-810cb95db16ea2b3418f12a23ced7304-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 810cb95db16ea2b3418f12a23ced7304 columnFamilyName testFamily 2024-11-08T11:19:41,681 DEBUG [StoreOpener-810cb95db16ea2b3418f12a23ced7304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:41,682 INFO [StoreOpener-810cb95db16ea2b3418f12a23ced7304-1 {}] regionserver.HStore(327): Store=810cb95db16ea2b3418f12a23ced7304/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:41,683 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,684 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,687 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:19:41,690 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:41,691 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 810cb95db16ea2b3418f12a23ced7304; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65005682, jitterRate=-0.0313398540019989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:41,692 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 810cb95db16ea2b3418f12a23ced7304: 2024-11-08T11:19:41,693 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304., pid=62, masterSystemTime=1731064781670 2024-11-08T11:19:41,694 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,694 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:19:41,695 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=810cb95db16ea2b3418f12a23ced7304, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:41,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-08T11:19:41,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 810cb95db16ea2b3418f12a23ced7304, server=ae52d81aa32e,33717,1731064629394 in 179 msec 2024-11-08T11:19:41,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T11:19:41,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=810cb95db16ea2b3418f12a23ced7304, ASSIGN in 335 msec 2024-11-08T11:19:41,701 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:41,701 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064781701"}]},"ts":"1731064781701"} 2024-11-08T11:19:41,703 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:41,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:41,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1830 sec 2024-11-08T11:19:41,984 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-08T11:19:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T11:19:43,530 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-08T11:19:43,530 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:43,530 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:43,535 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:43,535 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:43,535 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-08T11:19:43,537 DEBUG [hconnection-0x16a3e2db-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:43,538 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:43,547 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:43,550 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T11:19:43,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T11:19:43,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300146 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:43,553 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T11:19:43,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300146 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:43,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300146 2024-11-08T11:19:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:58382 deadline: 1731064786555, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:43,556 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:19:43,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T11:19:43,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300147 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:43,560 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T11:19:43,560 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:43,561 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:43,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300148 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:43,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4132839332912300148 2024-11-08T11:19:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:58382 deadline: 1731064786562, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:43,564 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:19:43,566 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:43,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300149 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:43,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4132839332912300149 2024-11-08T11:19:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:58382 deadline: 1731064786566, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:43,567 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:19:43,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:43,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300150 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:43,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4132839332912300150 2024-11-08T11:19:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:58382 deadline: 1731064786569, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T11:19:43,570 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:19:43,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T11:19:43,576 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1297d3a3 to 127.0.0.1:55518 2024-11-08T11:19:43,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:43,579 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:43,579 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:43,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:43,579 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3201081b to 127.0.0.1:55518 2024-11-08T11:19:43,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 326) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2453 (was 2465) 2024-11-08T11:19:43,602 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=340, ProcessCount=11, AvailableMemoryMB=2452 2024-11-08T11:19:43,603 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62ee8de9 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b392d37 2024-11-08T11:19:43,609 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x62ee8de9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17742a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:43,610 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b40394f to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a09a254 2024-11-08T11:19:43,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1ef939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:43,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T11:19:43,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:43,621 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:43,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-08T11:19:43,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T11:19:43,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741873_1049 (size=425) 2024-11-08T11:19:44,034 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => da78e98c858db35fcecc75c6fe00ecc8, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:44,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741874_1050 (size=76) 2024-11-08T11:19:44,442 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:44,442 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing da78e98c858db35fcecc75c6fe00ecc8, disabling compactions & flushes 2024-11-08T11:19:44,442 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,442 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,442 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. after waiting 0 ms 2024-11-08T11:19:44,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,443 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for da78e98c858db35fcecc75c6fe00ecc8: 2024-11-08T11:19:44,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:44,444 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731064784444"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064784444"}]},"ts":"1731064784444"} 2024-11-08T11:19:44,446 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:44,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:44,448 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064784447"}]},"ts":"1731064784447"} 2024-11-08T11:19:44,450 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:44,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=da78e98c858db35fcecc75c6fe00ecc8, ASSIGN}] 2024-11-08T11:19:44,456 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=da78e98c858db35fcecc75c6fe00ecc8, ASSIGN 2024-11-08T11:19:44,457 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=da78e98c858db35fcecc75c6fe00ecc8, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:44,607 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=da78e98c858db35fcecc75c6fe00ecc8, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:44,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure da78e98c858db35fcecc75c6fe00ecc8, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T11:19:44,761 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:44,766 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,767 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => da78e98c858db35fcecc75c6fe00ecc8, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:44,767 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,767 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:44,768 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,768 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,770 INFO [StoreOpener-da78e98c858db35fcecc75c6fe00ecc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,772 INFO [StoreOpener-da78e98c858db35fcecc75c6fe00ecc8-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da78e98c858db35fcecc75c6fe00ecc8 columnFamilyName testFamily 2024-11-08T11:19:44,772 DEBUG [StoreOpener-da78e98c858db35fcecc75c6fe00ecc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:44,773 INFO [StoreOpener-da78e98c858db35fcecc75c6fe00ecc8-1 {}] regionserver.HStore(327): Store=da78e98c858db35fcecc75c6fe00ecc8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:44,774 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,775 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,778 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:19:44,781 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:44,782 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened da78e98c858db35fcecc75c6fe00ecc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61285644, jitterRate=-0.08677273988723755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:44,782 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for da78e98c858db35fcecc75c6fe00ecc8: 2024-11-08T11:19:44,783 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8., pid=65, masterSystemTime=1731064784761 2024-11-08T11:19:44,785 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,786 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:19:44,786 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=da78e98c858db35fcecc75c6fe00ecc8, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:44,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T11:19:44,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure da78e98c858db35fcecc75c6fe00ecc8, server=ae52d81aa32e,33717,1731064629394 in 178 msec 2024-11-08T11:19:44,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-08T11:19:44,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=da78e98c858db35fcecc75c6fe00ecc8, ASSIGN in 334 msec 2024-11-08T11:19:44,792 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:44,792 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064784792"}]},"ts":"1731064784792"} 2024-11-08T11:19:44,794 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:44,796 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:44,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.1770 sec 2024-11-08T11:19:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T11:19:46,625 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-08T11:19:46,626 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:46,626 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:46,629 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:46,630 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:46,630 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T11:19:46,632 DEBUG [hconnection-0x48ac0a60-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:46,633 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58386, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:46,641 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:46,642 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:19:46,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:19:46,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T11:19:47,178 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:19:47,985 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-08T11:19:48,040 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300101 expired clientIPAndPort=172.17.0.2:37282, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:19:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T11:19:48,375 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:19:48,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:19:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:58386 deadline: 1731064789644, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:19:48,652 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:19:48,653 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:19:48,654 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2010 2024-11-08T11:19:48,654 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b40394f to 127.0.0.1:55518 2024-11-08T11:19:48,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:48,654 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:48,655 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:48,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:48,655 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62ee8de9 to 127.0.0.1:55518 2024-11-08T11:19:48,664 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=231 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=320 (was 340), ProcessCount=11 (was 11), AvailableMemoryMB=2433 (was 2452) 2024-11-08T11:19:48,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=320, ProcessCount=11, AvailableMemoryMB=2432 2024-11-08T11:19:48,674 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08afcff5 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c91eb01 2024-11-08T11:19:48,677 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x08afcff5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f2b30c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:48,678 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21f6a47c to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c5a1257 2024-11-08T11:19:48,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afca24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:48,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T11:19:48,685 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:48,685 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:48,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-08T11:19:48,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T11:19:48,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741875_1051 (size=426) 2024-11-08T11:19:49,096 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => fcce6bd67ff4fe800c5856874d71d076, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:49,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741876_1052 (size=77) 2024-11-08T11:19:49,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T11:19:49,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:49,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T11:19:49,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:49,504 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:49,504 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing fcce6bd67ff4fe800c5856874d71d076, disabling compactions & flushes 2024-11-08T11:19:49,504 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,504 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,504 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. after waiting 0 ms 2024-11-08T11:19:49,504 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,505 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,505 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for fcce6bd67ff4fe800c5856874d71d076: 2024-11-08T11:19:49,506 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:49,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731064789506"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064789506"}]},"ts":"1731064789506"} 2024-11-08T11:19:49,508 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:49,508 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:49,509 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064789508"}]},"ts":"1731064789508"} 2024-11-08T11:19:49,510 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:49,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=fcce6bd67ff4fe800c5856874d71d076, ASSIGN}] 2024-11-08T11:19:49,514 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=fcce6bd67ff4fe800c5856874d71d076, ASSIGN 2024-11-08T11:19:49,515 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=fcce6bd67ff4fe800c5856874d71d076, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:49,521 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:19:49,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=fcce6bd67ff4fe800c5856874d71d076, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:49,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure fcce6bd67ff4fe800c5856874d71d076, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T11:19:49,819 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:49,824 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,824 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => fcce6bd67ff4fe800c5856874d71d076, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:49,824 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,825 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:49,825 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,825 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,826 INFO [StoreOpener-fcce6bd67ff4fe800c5856874d71d076-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,828 INFO [StoreOpener-fcce6bd67ff4fe800c5856874d71d076-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcce6bd67ff4fe800c5856874d71d076 columnFamilyName testFamily 2024-11-08T11:19:49,828 DEBUG [StoreOpener-fcce6bd67ff4fe800c5856874d71d076-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:49,829 INFO [StoreOpener-fcce6bd67ff4fe800c5856874d71d076-1 {}] regionserver.HStore(327): Store=fcce6bd67ff4fe800c5856874d71d076/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:49,830 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,830 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,833 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:19:49,835 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:49,836 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened fcce6bd67ff4fe800c5856874d71d076; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65246094, jitterRate=-0.0277574360370636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:49,837 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for fcce6bd67ff4fe800c5856874d71d076: 2024-11-08T11:19:49,838 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076., pid=68, masterSystemTime=1731064789819 2024-11-08T11:19:49,839 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,839 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:19:49,840 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=fcce6bd67ff4fe800c5856874d71d076, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:49,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-08T11:19:49,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure fcce6bd67ff4fe800c5856874d71d076, server=ae52d81aa32e,33717,1731064629394 in 174 msec 2024-11-08T11:19:49,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T11:19:49,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=fcce6bd67ff4fe800c5856874d71d076, ASSIGN in 330 msec 2024-11-08T11:19:49,845 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:49,845 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064789845"}]},"ts":"1731064789845"} 2024-11-08T11:19:49,846 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:49,849 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:49,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1650 sec 2024-11-08T11:19:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T11:19:51,691 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-08T11:19:51,691 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:51,692 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:51,697 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:51,697 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:51,698 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T11:19:51,700 DEBUG [hconnection-0x1e925a08-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:51,701 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58596, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:51,712 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:51,714 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:19:51,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:19:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T11:19:53,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:19:53,987 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-08T11:19:54,717 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5019: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5019: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:19:54,718 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T11:19:54.717Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5019: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5019: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:19:54,718 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2024-11-08T11:19:54,719 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21f6a47c to 127.0.0.1:55518 2024-11-08T11:19:54,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:54,719 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:54,719 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:54,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:54,719 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08afcff5 to 127.0.0.1:55518 2024-11-08T11:19:54,733 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=231 (was 231), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=295 (was 320), ProcessCount=11 (was 11), AvailableMemoryMB=2415 (was 2432) 2024-11-08T11:19:54,746 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=2414 2024-11-08T11:19:54,750 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01858bfd to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c356263 2024-11-08T11:19:54,753 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x01858bfd {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295d7787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:54,755 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a3164c7 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61e96f7f 2024-11-08T11:19:54,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1edadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:54,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T11:19:54,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:54,765 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:54,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-08T11:19:54,766 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T11:19:54,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741877_1053 (size=424) 2024-11-08T11:19:55,201 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 805a942bf739e1744fdcc1a8ca9185fe, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:55,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741878_1054 (size=75) 2024-11-08T11:19:55,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:55,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 805a942bf739e1744fdcc1a8ca9185fe, disabling compactions & flushes 2024-11-08T11:19:55,213 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. after waiting 0 ms 2024-11-08T11:19:55,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,213 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 805a942bf739e1744fdcc1a8ca9185fe: 2024-11-08T11:19:55,217 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:55,217 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731064795217"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064795217"}]},"ts":"1731064795217"} 2024-11-08T11:19:55,219 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:55,220 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:55,220 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064795220"}]},"ts":"1731064795220"} 2024-11-08T11:19:55,222 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:55,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=805a942bf739e1744fdcc1a8ca9185fe, ASSIGN}] 2024-11-08T11:19:55,231 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=805a942bf739e1744fdcc1a8ca9185fe, ASSIGN 2024-11-08T11:19:55,231 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=805a942bf739e1744fdcc1a8ca9185fe, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:55,232 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:55,327 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:19:55,382 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=805a942bf739e1744fdcc1a8ca9185fe, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:55,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure 805a942bf739e1744fdcc1a8ca9185fe, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:55,537 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:55,542 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,542 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => 805a942bf739e1744fdcc1a8ca9185fe, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:55,543 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true 805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,543 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:55,543 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for 805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,543 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for 805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,546 INFO [StoreOpener-805a942bf739e1744fdcc1a8ca9185fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,548 INFO [StoreOpener-805a942bf739e1744fdcc1a8ca9185fe-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 805a942bf739e1744fdcc1a8ca9185fe columnFamilyName testFamily 2024-11-08T11:19:55,548 DEBUG [StoreOpener-805a942bf739e1744fdcc1a8ca9185fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:55,548 INFO [StoreOpener-805a942bf739e1744fdcc1a8ca9185fe-1 {}] regionserver.HStore(327): Store=805a942bf739e1744fdcc1a8ca9185fe/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:55,550 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,550 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,553 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for 805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:19:55,562 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:55,563 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened 805a942bf739e1744fdcc1a8ca9185fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63564317, jitterRate=-0.05281786620616913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:55,564 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for 805a942bf739e1744fdcc1a8ca9185fe: 2024-11-08T11:19:55,565 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe., pid=71, masterSystemTime=1731064795536 2024-11-08T11:19:55,567 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,567 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:19:55,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=805a942bf739e1744fdcc1a8ca9185fe, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:55,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T11:19:55,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure 805a942bf739e1744fdcc1a8ca9185fe, server=ae52d81aa32e,33717,1731064629394 in 185 msec 2024-11-08T11:19:55,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-08T11:19:55,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=805a942bf739e1744fdcc1a8ca9185fe, ASSIGN in 341 msec 2024-11-08T11:19:55,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:55,574 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064795574"}]},"ts":"1731064795574"} 2024-11-08T11:19:55,578 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:55,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:55,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 818 msec 2024-11-08T11:19:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T11:19:55,769 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-08T11:19:55,770 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:55,770 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:55,775 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:55,775 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:55,775 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T11:19:55,779 DEBUG [hconnection-0xa3339e5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:55,781 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58606, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:55,791 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:55,794 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:19:55,795 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:19:55,797 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x01858bfd {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:19:55,797 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x01858bfd {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:55,798 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:19:55,801 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:19:55,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300167 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:55,802 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:19:55,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300167 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:55,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T11:19:57,649 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300107 expired clientIPAndPort=172.17.0.2:45890, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:19:58,807 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3004 2024-11-08T11:19:58,808 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a3164c7 to 127.0.0.1:55518 2024-11-08T11:19:58,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:58,808 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:19:58,808 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300167 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:58,808 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:19:58,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:19:58,808 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01858bfd to 127.0.0.1:55518 2024-11-08T11:19:58,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300167' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T11:19:58,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:58614 deadline: 1731064799808, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4132839332912300167' does not exist 2024-11-08T11:19:58,812 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:58,809 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ae52d81aa32e,33717,1731064629394 for closing scanner id = 4132839332912300167 for 805a942bf739e1744fdcc1a8ca9185fe of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T11:19:58,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=232 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 295) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2375 (was 2414) 2024-11-08T11:19:58,836 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=232, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=2375 2024-11-08T11:19:58,838 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f0cf1cf to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4ba18a98 2024-11-08T11:19:58,842 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x4f0cf1cf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc40604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:58,843 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x443c6201 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1bae4a6c 2024-11-08T11:19:58,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3384425f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:19:58,852 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:19:58,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-08T11:19:58,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:19:58,855 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:58,855 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-08T11:19:58,856 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:19:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T11:19:58,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741879_1055 (size=416) 2024-11-08T11:19:58,870 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1ba55c509ca5439d425a248d570199f6, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:19:58,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741880_1056 (size=67) 2024-11-08T11:19:58,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:58,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1ba55c509ca5439d425a248d570199f6, disabling compactions & flushes 2024-11-08T11:19:58,892 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:58,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:58,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. after waiting 0 ms 2024-11-08T11:19:58,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:58,892 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:58,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1ba55c509ca5439d425a248d570199f6: 2024-11-08T11:19:58,893 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:19:58,894 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731064798894"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064798894"}]},"ts":"1731064798894"} 2024-11-08T11:19:58,896 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:19:58,897 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:19:58,897 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064798897"}]},"ts":"1731064798897"} 2024-11-08T11:19:58,899 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:19:58,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1ba55c509ca5439d425a248d570199f6, ASSIGN}] 2024-11-08T11:19:58,904 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1ba55c509ca5439d425a248d570199f6, ASSIGN 2024-11-08T11:19:58,905 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1ba55c509ca5439d425a248d570199f6, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:19:59,055 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=1ba55c509ca5439d425a248d570199f6, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:59,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 1ba55c509ca5439d425a248d570199f6, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:19:59,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T11:19:59,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:59,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T11:19:59,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:19:59,210 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:59,216 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:59,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 1ba55c509ca5439d425a248d570199f6, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:19:59,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:19:59,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,219 INFO [StoreOpener-1ba55c509ca5439d425a248d570199f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,221 INFO [StoreOpener-1ba55c509ca5439d425a248d570199f6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ba55c509ca5439d425a248d570199f6 columnFamilyName testFamily 2024-11-08T11:19:59,221 DEBUG [StoreOpener-1ba55c509ca5439d425a248d570199f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:19:59,222 INFO [StoreOpener-1ba55c509ca5439d425a248d570199f6-1 {}] regionserver.HStore(327): Store=1ba55c509ca5439d425a248d570199f6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:19:59,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 1ba55c509ca5439d425a248d570199f6 2024-11-08T11:19:59,230 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:19:59,231 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 1ba55c509ca5439d425a248d570199f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72553221, jitterRate=0.08112724125385284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:19:59,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 1ba55c509ca5439d425a248d570199f6: 2024-11-08T11:19:59,233 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6., pid=74, masterSystemTime=1731064799210 2024-11-08T11:19:59,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:59,235 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:19:59,236 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=1ba55c509ca5439d425a248d570199f6, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:19:59,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-08T11:19:59,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 1ba55c509ca5439d425a248d570199f6, server=ae52d81aa32e,33717,1731064629394 in 179 msec 2024-11-08T11:19:59,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T11:19:59,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1ba55c509ca5439d425a248d570199f6, ASSIGN in 336 msec 2024-11-08T11:19:59,242 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:19:59,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064799242"}]},"ts":"1731064799242"} 2024-11-08T11:19:59,244 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:19:59,247 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:19:59,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 395 msec 2024-11-08T11:19:59,303 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:19:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T11:19:59,859 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-08T11:19:59,859 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:19:59,859 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:59,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-08T11:19:59,866 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300168 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:19:59,867 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T11:19:59,867 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:19:59,867 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-08T11:19:59,869 DEBUG [hconnection-0x6884ba1a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:19:59,870 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38274, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:19:59,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:19:59,880 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:19:59,882 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300171 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:19:59,882 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T11:19:59,988 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-08T11:19:59,989 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-08T11:20:01,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:20:04,662 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300109 expired clientIPAndPort=172.17.0.2:45890, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:20:06,382 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-08T11:20:06,383 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x443c6201 to 127.0.0.1:55518 2024-11-08T11:20:06,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:06,383 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:06,383 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:06,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:06,383 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f0cf1cf to 127.0.0.1:55518 2024-11-08T11:20:06,395 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231 (was 232), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=338 (was 335) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2341 (was 2375) 2024-11-08T11:20:06,405 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=2340 2024-11-08T11:20:06,407 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c503948 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e4b4de 2024-11-08T11:20:06,411 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x6c503948 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cba1dd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:06,412 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b3216d6 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54e782a1 2024-11-08T11:20:06,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39106b0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:06,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:20:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-08T11:20:06,419 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:20:06,419 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:06,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-08T11:20:06,421 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:20:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T11:20:06,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741881_1057 (size=420) 2024-11-08T11:20:06,831 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 04d2e8f0a8c6ed33efc2158fe9bd9408, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:20:06,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741882_1058 (size=71) 2024-11-08T11:20:07,240 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:07,240 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 04d2e8f0a8c6ed33efc2158fe9bd9408, disabling compactions & flushes 2024-11-08T11:20:07,240 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,240 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,240 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. after waiting 0 ms 2024-11-08T11:20:07,240 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,240 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,241 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 04d2e8f0a8c6ed33efc2158fe9bd9408: 2024-11-08T11:20:07,242 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:20:07,243 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731064807242"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064807242"}]},"ts":"1731064807242"} 2024-11-08T11:20:07,244 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:20:07,245 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:20:07,245 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064807245"}]},"ts":"1731064807245"} 2024-11-08T11:20:07,247 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:20:07,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=04d2e8f0a8c6ed33efc2158fe9bd9408, ASSIGN}] 2024-11-08T11:20:07,255 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=04d2e8f0a8c6ed33efc2158fe9bd9408, ASSIGN 2024-11-08T11:20:07,255 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=04d2e8f0a8c6ed33efc2158fe9bd9408, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:20:07,406 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=04d2e8f0a8c6ed33efc2158fe9bd9408, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:07,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 04d2e8f0a8c6ed33efc2158fe9bd9408, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:20:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T11:20:07,561 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:07,566 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,567 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => 04d2e8f0a8c6ed33efc2158fe9bd9408, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:20:07,567 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,567 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:07,567 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for 04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,567 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for 04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,569 INFO [StoreOpener-04d2e8f0a8c6ed33efc2158fe9bd9408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,571 INFO [StoreOpener-04d2e8f0a8c6ed33efc2158fe9bd9408-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04d2e8f0a8c6ed33efc2158fe9bd9408 columnFamilyName testFamily 2024-11-08T11:20:07,571 DEBUG [StoreOpener-04d2e8f0a8c6ed33efc2158fe9bd9408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:07,572 INFO [StoreOpener-04d2e8f0a8c6ed33efc2158fe9bd9408-1 {}] regionserver.HStore(327): Store=04d2e8f0a8c6ed33efc2158fe9bd9408/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:20:07,573 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,574 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,576 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for 04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:07,579 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:20:07,580 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened 04d2e8f0a8c6ed33efc2158fe9bd9408; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66918679, jitterRate=-0.0028339773416519165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:20:07,581 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for 04d2e8f0a8c6ed33efc2158fe9bd9408: 2024-11-08T11:20:07,582 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408., pid=77, masterSystemTime=1731064807561 2024-11-08T11:20:07,584 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,584 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:20:07,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=04d2e8f0a8c6ed33efc2158fe9bd9408, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:07,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T11:20:07,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 04d2e8f0a8c6ed33efc2158fe9bd9408, server=ae52d81aa32e,33717,1731064629394 in 177 msec 2024-11-08T11:20:07,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-08T11:20:07,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=04d2e8f0a8c6ed33efc2158fe9bd9408, ASSIGN in 335 msec 2024-11-08T11:20:07,590 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:20:07,590 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064807590"}]},"ts":"1731064807590"} 2024-11-08T11:20:07,592 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:20:07,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:20:07,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1780 sec 2024-11-08T11:20:07,772 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:20:09,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T11:20:09,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T11:20:09,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T11:20:09,424 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-08T11:20:09,424 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:20:09,424 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:09,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-08T11:20:09,430 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300172 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:09,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T11:20:09,432 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:09,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-08T11:20:09,434 DEBUG [hconnection-0x22d595a1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:09,435 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38280, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:09,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:20:09,446 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:20:09,449 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300175 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T11:20:09,449 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T11:20:09,767 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300114 expired clientIPAndPort=172.17.0.2:36592, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:20:10,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-08T11:20:10,953 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b3216d6 to 127.0.0.1:55518 2024-11-08T11:20:10,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:10,953 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:10,953 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:10,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:10,953 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c503948 to 127.0.0.1:55518 2024-11-08T11:20:10,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 338), ProcessCount=11 (was 11), AvailableMemoryMB=2327 (was 2340) 2024-11-08T11:20:10,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=2326 2024-11-08T11:20:10,977 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3770b12c to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26a91b1 2024-11-08T11:20:11,003 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x3770b12c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648be025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:11,004 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27b65a51 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3eb73602 2024-11-08T11:20:11,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7c1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:11,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:20:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-08T11:20:11,013 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:20:11,013 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:11,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-08T11:20:11,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:20:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T11:20:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741883_1059 (size=419) 2024-11-08T11:20:11,429 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9e41cf0f49a0c415a7312fe4fabd7166, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:20:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741884_1060 (size=70) 2024-11-08T11:20:11,837 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:11,837 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 9e41cf0f49a0c415a7312fe4fabd7166, disabling compactions & flushes 2024-11-08T11:20:11,837 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:11,837 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:11,837 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. after waiting 0 ms 2024-11-08T11:20:11,837 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:11,837 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:11,837 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9e41cf0f49a0c415a7312fe4fabd7166: 2024-11-08T11:20:11,838 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:20:11,838 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731064811838"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064811838"}]},"ts":"1731064811838"} 2024-11-08T11:20:11,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:20:11,840 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:20:11,841 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064811841"}]},"ts":"1731064811841"} 2024-11-08T11:20:11,842 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:20:11,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9e41cf0f49a0c415a7312fe4fabd7166, ASSIGN}] 2024-11-08T11:20:11,847 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9e41cf0f49a0c415a7312fe4fabd7166, ASSIGN 2024-11-08T11:20:11,847 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9e41cf0f49a0c415a7312fe4fabd7166, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:20:11,991 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-08T11:20:11,998 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=9e41cf0f49a0c415a7312fe4fabd7166, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:12,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 9e41cf0f49a0c415a7312fe4fabd7166, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:20:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T11:20:12,151 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:12,156 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:12,156 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => 9e41cf0f49a0c415a7312fe4fabd7166, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:20:12,156 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,156 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:12,156 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for 9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,156 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for 9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,158 INFO [StoreOpener-9e41cf0f49a0c415a7312fe4fabd7166-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,160 INFO [StoreOpener-9e41cf0f49a0c415a7312fe4fabd7166-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e41cf0f49a0c415a7312fe4fabd7166 columnFamilyName testFamily 2024-11-08T11:20:12,160 DEBUG [StoreOpener-9e41cf0f49a0c415a7312fe4fabd7166-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:12,161 INFO [StoreOpener-9e41cf0f49a0c415a7312fe4fabd7166-1 {}] regionserver.HStore(327): Store=9e41cf0f49a0c415a7312fe4fabd7166/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:20:12,162 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,162 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,164 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for 9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:12,166 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:20:12,167 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened 9e41cf0f49a0c415a7312fe4fabd7166; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74120317, jitterRate=0.10447879135608673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:20:12,168 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for 9e41cf0f49a0c415a7312fe4fabd7166: 2024-11-08T11:20:12,168 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166., pid=80, masterSystemTime=1731064812151 2024-11-08T11:20:12,170 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:12,170 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:12,171 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=9e41cf0f49a0c415a7312fe4fabd7166, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:12,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-08T11:20:12,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 9e41cf0f49a0c415a7312fe4fabd7166, server=ae52d81aa32e,33717,1731064629394 in 173 msec 2024-11-08T11:20:12,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T11:20:12,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9e41cf0f49a0c415a7312fe4fabd7166, ASSIGN in 327 msec 2024-11-08T11:20:12,175 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:20:12,175 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064812175"}]},"ts":"1731064812175"} 2024-11-08T11:20:12,176 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:20:12,179 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:20:12,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1680 sec 2024-11-08T11:20:13,108 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:20:13,770 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300119 expired clientIPAndPort=172.17.0.2:54502, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:20:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T11:20:14,018 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-08T11:20:14,018 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:20:14,018 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:14,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-08T11:20:14,023 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300176 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:14,024 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T11:20:14,024 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:14,024 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-08T11:20:14,027 DEBUG [hconnection-0x11ad717c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:14,028 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47066, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:14,035 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:20:14,037 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:20:14,041 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:20:14,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300179 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:14,042 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:20:14,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300179 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:14,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T11:20:15,872 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300121 expired clientIPAndPort=172.17.0.2:54502, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:20:17,047 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4013: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4013: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:20:17,047 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3004 2024-11-08T11:20:17,047 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x27b65a51 to 127.0.0.1:55518 2024-11-08T11:20:17,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:17,048 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:17,048 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:17,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:17,048 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3770b12c to 127.0.0.1:55518 2024-11-08T11:20:17,057 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=2284 (was 2326) 2024-11-08T11:20:17,065 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=2284 2024-11-08T11:20:17,067 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1597e5a0 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@785308bb 2024-11-08T11:20:17,070 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x1597e5a0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca43325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:17,071 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6aabaf72 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@335965c4 2024-11-08T11:20:17,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19493cea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:17,077 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:20:17,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T11:20:17,079 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:20:17,079 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:17,079 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-08T11:20:17,079 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:20:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T11:20:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741885_1061 (size=421) 2024-11-08T11:20:17,489 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3ecc4bd45e63090de8e9f1621e42d4d0, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:20:17,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741886_1062 (size=72) 2024-11-08T11:20:17,544 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:20:17,897 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:17,897 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 3ecc4bd45e63090de8e9f1621e42d4d0, disabling compactions & flushes 2024-11-08T11:20:17,897 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:17,897 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:17,897 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. after waiting 0 ms 2024-11-08T11:20:17,897 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:17,897 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:17,897 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3ecc4bd45e63090de8e9f1621e42d4d0: 2024-11-08T11:20:17,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:20:17,898 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731064817898"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064817898"}]},"ts":"1731064817898"} 2024-11-08T11:20:17,900 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:20:17,900 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:20:17,901 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064817900"}]},"ts":"1731064817900"} 2024-11-08T11:20:17,902 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:20:17,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=3ecc4bd45e63090de8e9f1621e42d4d0, ASSIGN}] 2024-11-08T11:20:17,906 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=3ecc4bd45e63090de8e9f1621e42d4d0, ASSIGN 2024-11-08T11:20:17,906 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=3ecc4bd45e63090de8e9f1621e42d4d0, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:20:17,993 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-08T11:20:18,057 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=3ecc4bd45e63090de8e9f1621e42d4d0, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:18,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 3ecc4bd45e63090de8e9f1621e42d4d0, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:20:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T11:20:18,211 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:18,215 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:18,215 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => 3ecc4bd45e63090de8e9f1621e42d4d0, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:20:18,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:18,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for 3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for 3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,218 INFO [StoreOpener-3ecc4bd45e63090de8e9f1621e42d4d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,219 INFO [StoreOpener-3ecc4bd45e63090de8e9f1621e42d4d0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ecc4bd45e63090de8e9f1621e42d4d0 columnFamilyName testFamily 2024-11-08T11:20:18,219 DEBUG [StoreOpener-3ecc4bd45e63090de8e9f1621e42d4d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:18,220 INFO [StoreOpener-3ecc4bd45e63090de8e9f1621e42d4d0-1 {}] regionserver.HStore(327): Store=3ecc4bd45e63090de8e9f1621e42d4d0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:20:18,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for 3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:18,225 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:20:18,226 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened 3ecc4bd45e63090de8e9f1621e42d4d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63022524, jitterRate=-0.06089121103286743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:20:18,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for 3ecc4bd45e63090de8e9f1621e42d4d0: 2024-11-08T11:20:18,227 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0., pid=83, masterSystemTime=1731064818211 2024-11-08T11:20:18,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:18,229 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:18,229 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=3ecc4bd45e63090de8e9f1621e42d4d0, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:18,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T11:20:18,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 3ecc4bd45e63090de8e9f1621e42d4d0, server=ae52d81aa32e,33717,1731064629394 in 172 msec 2024-11-08T11:20:18,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T11:20:18,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=3ecc4bd45e63090de8e9f1621e42d4d0, ASSIGN in 327 msec 2024-11-08T11:20:18,234 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:20:18,234 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064818234"}]},"ts":"1731064818234"} 2024-11-08T11:20:18,236 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:20:18,238 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:20:18,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1610 sec 2024-11-08T11:20:18,692 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:20:19,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T11:20:19,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:19,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T11:20:19,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:19,837 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:20:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T11:20:20,083 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-08T11:20:20,083 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:20:20,083 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:20,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-08T11:20:20,087 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300180 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:20,088 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T11:20:20,088 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:20,088 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T11:20:20,090 DEBUG [hconnection-0x605da100-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:20,092 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51972, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:20,099 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:20:20,100 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:20:20,102 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:20:20,102 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T11:20:23,757 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:20:23,994 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-08T11:20:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T11:20:27,116 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:20:27,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51972 deadline: 1731064829102, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T11:20:27,117 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T11:20:27,117 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4132839332912300183 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T11:20:27,118 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7017 2024-11-08T11:20:27,118 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6aabaf72 to 127.0.0.1:55518 2024-11-08T11:20:27,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:27,118 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:27,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:27,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:27,119 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1597e5a0 to 127.0.0.1:55518 2024-11-08T11:20:27,129 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=228 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=2102 (was 2284) 2024-11-08T11:20:27,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=228, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=2101 2024-11-08T11:20:27,140 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59e144eb to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27d32501 2024-11-08T11:20:27,178 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x59e144eb {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4167654a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:27,179 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0188d79c to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3df02d7e 2024-11-08T11:20:27,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e65e037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:27,184 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:20:27,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T11:20:27,187 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:20:27,187 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:27,187 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-08T11:20:27,188 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:20:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T11:20:27,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741887_1063 (size=427) 2024-11-08T11:20:27,603 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 967fd2a06121897b47c8cd11f2da98b3, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:20:27,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741888_1064 (size=78) 2024-11-08T11:20:27,616 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:27,616 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 967fd2a06121897b47c8cd11f2da98b3, disabling compactions & flushes 2024-11-08T11:20:27,616 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,616 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,616 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. after waiting 0 ms 2024-11-08T11:20:27,616 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,616 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,616 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 967fd2a06121897b47c8cd11f2da98b3: 2024-11-08T11:20:27,617 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:20:27,618 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731064827617"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064827617"}]},"ts":"1731064827617"} 2024-11-08T11:20:27,620 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:20:27,621 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:20:27,621 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064827621"}]},"ts":"1731064827621"} 2024-11-08T11:20:27,622 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:20:27,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=967fd2a06121897b47c8cd11f2da98b3, ASSIGN}] 2024-11-08T11:20:27,628 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=967fd2a06121897b47c8cd11f2da98b3, ASSIGN 2024-11-08T11:20:27,629 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=967fd2a06121897b47c8cd11f2da98b3, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:20:27,780 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=967fd2a06121897b47c8cd11f2da98b3, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:27,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure 967fd2a06121897b47c8cd11f2da98b3, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:20:27,934 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:27,940 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => 967fd2a06121897b47c8cd11f2da98b3, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:20:27,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true 967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:27,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for 967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for 967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,946 INFO [StoreOpener-967fd2a06121897b47c8cd11f2da98b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,950 INFO [StoreOpener-967fd2a06121897b47c8cd11f2da98b3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 967fd2a06121897b47c8cd11f2da98b3 columnFamilyName testFamily 2024-11-08T11:20:27,950 DEBUG [StoreOpener-967fd2a06121897b47c8cd11f2da98b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:27,951 INFO [StoreOpener-967fd2a06121897b47c8cd11f2da98b3-1 {}] regionserver.HStore(327): Store=967fd2a06121897b47c8cd11f2da98b3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:20:27,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for 967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:27,959 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:20:27,959 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened 967fd2a06121897b47c8cd11f2da98b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69260866, jitterRate=0.032067328691482544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:20:27,960 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for 967fd2a06121897b47c8cd11f2da98b3: 2024-11-08T11:20:27,962 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., pid=86, masterSystemTime=1731064827934 2024-11-08T11:20:27,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,964 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:20:27,964 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=967fd2a06121897b47c8cd11f2da98b3, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:27,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T11:20:27,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure 967fd2a06121897b47c8cd11f2da98b3, server=ae52d81aa32e,33717,1731064629394 in 183 msec 2024-11-08T11:20:27,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T11:20:27,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=967fd2a06121897b47c8cd11f2da98b3, ASSIGN in 340 msec 2024-11-08T11:20:27,973 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:20:27,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064827973"}]},"ts":"1731064827973"} 2024-11-08T11:20:27,975 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:20:27,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:20:27,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 795 msec 2024-11-08T11:20:27,982 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300135 expired clientIPAndPort=172.17.0.2:38106, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:20:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T11:20:28,190 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-08T11:20:28,190 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:20:28,191 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:28,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-08T11:20:28,203 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300185 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:28,204 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T11:20:28,205 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:28,205 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T11:20:28,213 DEBUG [hconnection-0x3bfebf3e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:28,215 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51986, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:28,226 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:20:28,234 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:20:28,235 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:20:28,236 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x59e144eb {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:20:28,236 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x59e144eb {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:28,237 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51994, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:20:28,240 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:20:29,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T11:20:29,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:29,996 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T11:20:33,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:20:34,247 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms 2024-11-08T11:20:34,247 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms 2024-11-08T11:20:34,247 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:20:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T11:20:34,988 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300137 expired clientIPAndPort=172.17.0.2:38106, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:20:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:20:35,258 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T11:20:37,237 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms 2024-11-08T11:20:37,237 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms 2024-11-08T11:20:37,237 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:20:37,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T11:20:34.247Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms 2024-11-08T11:20:37.238Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T11:20:37,239 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2024-11-08T11:20:37,239 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0188d79c to 127.0.0.1:55518 2024-11-08T11:20:37,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:37,240 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:37,240 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:37,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:37,240 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59e144eb to 127.0.0.1:55518 2024-11-08T11:20:37,249 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230 (was 228) - Thread LEAK? -, OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 268) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2611 (was 2101) - AvailableMemoryMB LEAK? - 2024-11-08T11:20:37,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=2610 2024-11-08T11:20:37,259 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x789d9a0f to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e97173d 2024-11-08T11:20:37,399 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x789d9a0f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777f2809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:37,400 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3195f2ae to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b3b8455 2024-11-08T11:20:37,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe23cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:37,449 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:20:37,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-08T11:20:37,451 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:20:37,451 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:37,451 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-08T11:20:37,451 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:20:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T11:20:37,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741889_1065 (size=415) 2024-11-08T11:20:37,772 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:20:37,861 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c1a5df7938043bbc4d0adc89c260dff6, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:20:37,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741890_1066 (size=66) 2024-11-08T11:20:38,269 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:38,269 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing c1a5df7938043bbc4d0adc89c260dff6, disabling compactions & flushes 2024-11-08T11:20:38,269 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,270 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,270 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. after waiting 0 ms 2024-11-08T11:20:38,270 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,270 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,270 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for c1a5df7938043bbc4d0adc89c260dff6: 2024-11-08T11:20:38,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:20:38,271 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731064838271"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064838271"}]},"ts":"1731064838271"} 2024-11-08T11:20:38,273 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:20:38,273 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:20:38,274 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064838274"}]},"ts":"1731064838274"} 2024-11-08T11:20:38,275 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:20:38,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=c1a5df7938043bbc4d0adc89c260dff6, ASSIGN}] 2024-11-08T11:20:38,347 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=c1a5df7938043bbc4d0adc89c260dff6, ASSIGN 2024-11-08T11:20:38,347 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=c1a5df7938043bbc4d0adc89c260dff6, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:20:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T11:20:38,498 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=c1a5df7938043bbc4d0adc89c260dff6, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:38,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure c1a5df7938043bbc4d0adc89c260dff6, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:20:38,652 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:38,658 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => c1a5df7938043bbc4d0adc89c260dff6, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:20:38,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:38,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,660 INFO [StoreOpener-c1a5df7938043bbc4d0adc89c260dff6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,662 INFO [StoreOpener-c1a5df7938043bbc4d0adc89c260dff6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1a5df7938043bbc4d0adc89c260dff6 columnFamilyName testFamily 2024-11-08T11:20:38,662 DEBUG [StoreOpener-c1a5df7938043bbc4d0adc89c260dff6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:38,663 INFO [StoreOpener-c1a5df7938043bbc4d0adc89c260dff6-1 {}] regionserver.HStore(327): Store=c1a5df7938043bbc4d0adc89c260dff6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:20:38,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,666 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:38,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:20:38,669 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened c1a5df7938043bbc4d0adc89c260dff6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63745669, jitterRate=-0.05011551082134247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:20:38,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for c1a5df7938043bbc4d0adc89c260dff6: 2024-11-08T11:20:38,670 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6., pid=89, masterSystemTime=1731064838652 2024-11-08T11:20:38,671 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,671 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:38,672 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=c1a5df7938043bbc4d0adc89c260dff6, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:38,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T11:20:38,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure c1a5df7938043bbc4d0adc89c260dff6, server=ae52d81aa32e,33717,1731064629394 in 173 msec 2024-11-08T11:20:38,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-08T11:20:38,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=c1a5df7938043bbc4d0adc89c260dff6, ASSIGN in 330 msec 2024-11-08T11:20:38,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:20:38,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064838678"}]},"ts":"1731064838678"} 2024-11-08T11:20:38,679 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:20:38,682 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:20:38,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.2330 sec 2024-11-08T11:20:39,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T11:20:39,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T11:20:40,455 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-08T11:20:40,455 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:20:40,455 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:40,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-08T11:20:40,459 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300192 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:40,460 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T11:20:40,461 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:40,461 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-08T11:20:40,462 DEBUG [hconnection-0x1023b6aa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:40,463 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49292, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:40,470 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:20:40,471 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T11:20:40,473 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T11:20:40,473 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300195 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:40,474 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T11:20:40,474 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300195 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:40,474 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T11:20:40,994 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300142 expired clientIPAndPort=172.17.0.2:38858, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:20:41,759 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:20:41,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-08T11:20:43,597 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300146 expired clientIPAndPort=172.17.0.2:58382, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:43,597 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300149 expired clientIPAndPort=172.17.0.2:58382, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:43,597 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300148 expired clientIPAndPort=172.17.0.2:58382, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:44,199 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:20:48,701 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300155 expired clientIPAndPort=172.17.0.2:58386, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:20:49,477 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T11:20:49,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9003 2024-11-08T11:20:49,477 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3195f2ae to 127.0.0.1:55518 2024-11-08T11:20:49,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:49,478 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:49,478 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:49,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:49,478 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x789d9a0f to 127.0.0.1:55518 2024-11-08T11:20:49,487 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=2495 (was 2610) 2024-11-08T11:20:49,496 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=2495 2024-11-08T11:20:49,498 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x623aea41 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74ed87fe 2024-11-08T11:20:49,501 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x623aea41 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bd6a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:49,502 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x756f7267 to 127.0.0.1:55518 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@608016b3 2024-11-08T11:20:49,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555d9000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T11:20:49,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T11:20:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T11:20:49,509 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T11:20:49,509 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:49,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-08T11:20:49,510 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T11:20:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T11:20:49,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741891_1067 (size=431) 2024-11-08T11:20:49,920 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f2b790a2329bc0309d783fbb87b96ddd, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a 2024-11-08T11:20:49,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741892_1068 (size=82) 2024-11-08T11:20:49,975 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:20:50,328 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:50,328 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f2b790a2329bc0309d783fbb87b96ddd, disabling compactions & flushes 2024-11-08T11:20:50,328 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,328 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,328 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. after waiting 0 ms 2024-11-08T11:20:50,328 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,328 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,328 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f2b790a2329bc0309d783fbb87b96ddd: 2024-11-08T11:20:50,329 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T11:20:50,329 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731064850329"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731064850329"}]},"ts":"1731064850329"} 2024-11-08T11:20:50,331 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T11:20:50,332 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T11:20:50,332 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064850332"}]},"ts":"1731064850332"} 2024-11-08T11:20:50,333 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T11:20:50,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=f2b790a2329bc0309d783fbb87b96ddd, ASSIGN}] 2024-11-08T11:20:50,338 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=f2b790a2329bc0309d783fbb87b96ddd, ASSIGN 2024-11-08T11:20:50,338 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=f2b790a2329bc0309d783fbb87b96ddd, ASSIGN; state=OFFLINE, location=ae52d81aa32e,33717,1731064629394; forceNewPlan=false, retain=false 2024-11-08T11:20:50,489 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=f2b790a2329bc0309d783fbb87b96ddd, regionState=OPENING, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:50,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure f2b790a2329bc0309d783fbb87b96ddd, server=ae52d81aa32e,33717,1731064629394}] 2024-11-08T11:20:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T11:20:50,642 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:50,648 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,648 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => f2b790a2329bc0309d783fbb87b96ddd, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd.', STARTKEY => '', ENDKEY => ''} 2024-11-08T11:20:50,648 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,648 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T11:20:50,649 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,649 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,650 INFO [StoreOpener-f2b790a2329bc0309d783fbb87b96ddd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,653 INFO [StoreOpener-f2b790a2329bc0309d783fbb87b96ddd-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2b790a2329bc0309d783fbb87b96ddd columnFamilyName testFamily 2024-11-08T11:20:50,654 DEBUG [StoreOpener-f2b790a2329bc0309d783fbb87b96ddd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T11:20:50,654 INFO [StoreOpener-f2b790a2329bc0309d783fbb87b96ddd-1 {}] regionserver.HStore(327): Store=f2b790a2329bc0309d783fbb87b96ddd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T11:20:50,655 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,656 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,658 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:50,661 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T11:20:50,661 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened f2b790a2329bc0309d783fbb87b96ddd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59079510, jitterRate=-0.11964669823646545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T11:20:50,662 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for f2b790a2329bc0309d783fbb87b96ddd: 2024-11-08T11:20:50,663 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., pid=92, masterSystemTime=1731064850642 2024-11-08T11:20:50,665 DEBUG [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,665 INFO [RS_OPEN_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:20:50,665 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=f2b790a2329bc0309d783fbb87b96ddd, regionState=OPEN, openSeqNum=2, regionLocation=ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:50,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-08T11:20:50,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure f2b790a2329bc0309d783fbb87b96ddd, server=ae52d81aa32e,33717,1731064629394 in 175 msec 2024-11-08T11:20:50,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T11:20:50,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=f2b790a2329bc0309d783fbb87b96ddd, ASSIGN in 330 msec 2024-11-08T11:20:50,669 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T11:20:50,669 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731064850669"}]},"ts":"1731064850669"} 2024-11-08T11:20:50,670 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T11:20:50,673 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T11:20:50,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1650 sec 2024-11-08T11:20:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36127 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T11:20:52,513 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-08T11:20:52,513 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T11:20:52,514 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:52,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33717 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-08T11:20:52,517 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4132839332912300196 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T11:20:52,518 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T11:20:52,518 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T11:20:52,518 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T11:20:52,520 DEBUG [hconnection-0x6e4e284d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:52,521 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:52,528 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T11:20:52,530 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T11:20:52,530 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T11:20:52,531 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x623aea41 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae52d81aa32e,33717,1731064629394, seqNum=-1] 2024-11-08T11:20:52,531 DEBUG [ReadOnlyZKClient-127.0.0.1:55518@0x623aea41 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T11:20:52,532 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46616, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T11:20:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:20:52,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:20:53,304 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300160 expired clientIPAndPort=172.17.0.2:58596, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:20:53,537 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-08T11:20:53,537 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-08T11:20:53,537 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:20:53,999 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T11:20:54,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T11:20:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2] 2024-11-08T11:20:54,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T11:20:55,306 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300162 expired clientIPAndPort=172.17.0.2:58596, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:20:55,537 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=990ms, rpcTimeout=983ms 2024-11-08T11:20:55,537 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=990ms, rpcTimeout=983ms 2024-11-08T11:20:55,537 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd., hostname=ae52d81aa32e,33717,1731064629394, seqNum=2 from cache 2024-11-08T11:20:55,538 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T11:20:53.537Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-08T11:20:55.537Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=990ms, rpcTimeout=983ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ae52d81aa32e:33717 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=990ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=990ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T11:20:55,539 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3008 2024-11-08T11:20:55,539 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x756f7267 to 127.0.0.1:55518 2024-11-08T11:20:55,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:55,539 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T11:20:55,539 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T11:20:55,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:55,540 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x623aea41 to 127.0.0.1:55518 2024-11-08T11:20:55,549 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=230 (was 226) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=2463 (was 2495) 2024-11-08T11:20:55,549 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-08T11:20:55,549 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T11:20:55,549 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x574a453f to 127.0.0.1:55518 2024-11-08T11:20:55,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:55,549 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T11:20:55,549 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1111814696, stopped=false 2024-11-08T11:20:55,549 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ae52d81aa32e,36127,1731064628628 2024-11-08T11:20:55,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T11:20:55,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T11:20:55,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:20:55,551 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-08T11:20:55,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:20:55,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:55,552 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ae52d81aa32e,33717,1731064629394' ***** 2024-11-08T11:20:55,552 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-08T11:20:55,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T11:20:55,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T11:20:55,552 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T11:20:55,552 INFO [RS:0;ae52d81aa32e:33717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T11:20:55,552 INFO [RS:0;ae52d81aa32e:33717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T11:20:55,552 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-08T11:20:55,552 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 9e41cf0f49a0c415a7312fe4fabd7166 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 8854a0dae2dc4ae8fc413fa10c0ab93c 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for c1a5df7938043bbc4d0adc89c260dff6 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for a198a6c470d9a189e91f0680bf15f8f4 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for decbab74a7be3e3a5b25bee2e10ab413 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 47efa016b96d371d1d5a39cfe0614766 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 1ba55c509ca5439d425a248d570199f6 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 3ecc4bd45e63090de8e9f1621e42d4d0 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 28a413aad77fe00c80109f757f9eff1d 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 810cb95db16ea2b3418f12a23ced7304 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 27c4e829c2cf184f5508de640a25b0bb 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 04d2e8f0a8c6ed33efc2158fe9bd9408 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 8c25b44c431bf3f90b6b41bc98a7e29d 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 8029e11915c8ac0360de37aa6659111e 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for ce6c8c461db85938ff504cfbc5c00973 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for b96aee56c445cd36471b3c7201edc90d 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for f2b790a2329bc0309d783fbb87b96ddd 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for b6492f1f8d3b65f481a71f69b3f63ffa 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for e187798f3f2dc14ef76d302202186f7c 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for ad75048110dcede8a9d5403d1e5644c9 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 967fd2a06121897b47c8cd11f2da98b3 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 805a942bf739e1744fdcc1a8ca9185fe 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for da78e98c858db35fcecc75c6fe00ecc8 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 012d9e50c5a5dfe248e1e5d5c6aa397b 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 2ae2b67ac99527ab93cd9d51d12df9c3 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for a97e8684ef3557cc5c8111e42191ba0f 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 0aa76efa23785cc8435444278f71e915 2024-11-08T11:20:55,553 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 7fdf3e5624b7e358e192563c0644e9a1 2024-11-08T11:20:55,553 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9e41cf0f49a0c415a7312fe4fabd7166, disabling compactions & flushes 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1224): stopping server ae52d81aa32e,33717,1731064629394 2024-11-08T11:20:55,554 DEBUG [RS:0;ae52d81aa32e:33717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:20:55,554 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. after waiting 0 ms 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-08T11:20:55,554 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T11:20:55,554 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9e41cf0f49a0c415a7312fe4fabd7166 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:55,554 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T11:20:55,554 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T11:20:55,554 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-08T11:20:55,555 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1588230740, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8854a0dae2dc4ae8fc413fa10c0ab93c, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, 9e41cf0f49a0c415a7312fe4fabd7166, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:55,592 INFO [regionserver/ae52d81aa32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T11:20:55,592 INFO [regionserver/ae52d81aa32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T11:20:55,610 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/.tmp/testFamily/9bf0dd5a3f614220abd2958789e18916 is 61, key is row-0/testFamily:testQualifier/1731064814030/Put/seqid=0 2024-11-08T11:20:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741893_1069 (size=5049) 2024-11-08T11:20:55,620 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/.tmp/info/bdac9c041f774091ac32c7894c574456 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3./info:regioninfo/1731064664996/Put/seqid=0 2024-11-08T11:20:55,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741894_1070 (size=51873) 2024-11-08T11:20:55,755 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1588230740, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8854a0dae2dc4ae8fc413fa10c0ab93c, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, 9e41cf0f49a0c415a7312fe4fabd7166, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:55,955 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1588230740, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8854a0dae2dc4ae8fc413fa10c0ab93c, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, 9e41cf0f49a0c415a7312fe4fabd7166, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:56,021 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/.tmp/testFamily/9bf0dd5a3f614220abd2958789e18916 2024-11-08T11:20:56,025 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/.tmp/info/bdac9c041f774091ac32c7894c574456 2024-11-08T11:20:56,048 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T11:20:56,065 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/.tmp/testFamily/9bf0dd5a3f614220abd2958789e18916 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/testFamily/9bf0dd5a3f614220abd2958789e18916 2024-11-08T11:20:56,072 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/testFamily/9bf0dd5a3f614220abd2958789e18916, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:56,076 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9e41cf0f49a0c415a7312fe4fabd7166 in 522ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:56,080 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_1_-true/9e41cf0f49a0c415a7312fe4fabd7166/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:56,081 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/.tmp/table/5baa896784ef4cf9a56f45a0bc36ea3e is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731064665008/Put/seqid=0 2024-11-08T11:20:56,083 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:56,083 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9e41cf0f49a0c415a7312fe4fabd7166: 2024-11-08T11:20:56,083 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731064811010.9e41cf0f49a0c415a7312fe4fabd7166. 2024-11-08T11:20:56,083 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8854a0dae2dc4ae8fc413fa10c0ab93c, disabling compactions & flushes 2024-11-08T11:20:56,083 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:20:56,083 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:20:56,083 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. after waiting 0 ms 2024-11-08T11:20:56,083 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:20:56,083 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8854a0dae2dc4ae8fc413fa10c0ab93c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:56,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741895_1071 (size=9821) 2024-11-08T11:20:56,098 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/.tmp/testFamily/32fc1396e26f4ac2a1da022ea2a8cd70 is 61, key is row-0/testFamily:testQualifier/1731064720972/Put/seqid=0 2024-11-08T11:20:56,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741896_1072 (size=5049) 2024-11-08T11:20:56,155 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1588230740, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8854a0dae2dc4ae8fc413fa10c0ab93c, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:56,184 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:20:56,356 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1588230740, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8854a0dae2dc4ae8fc413fa10c0ab93c, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:56,487 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/.tmp/table/5baa896784ef4cf9a56f45a0bc36ea3e 2024-11-08T11:20:56,495 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/.tmp/info/bdac9c041f774091ac32c7894c574456 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/info/bdac9c041f774091ac32c7894c574456 2024-11-08T11:20:56,501 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/info/bdac9c041f774091ac32c7894c574456, entries=290, sequenceid=149, filesize=50.7 K 2024-11-08T11:20:56,502 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/.tmp/table/5baa896784ef4cf9a56f45a0bc36ea3e as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/table/5baa896784ef4cf9a56f45a0bc36ea3e 2024-11-08T11:20:56,502 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/.tmp/testFamily/32fc1396e26f4ac2a1da022ea2a8cd70 2024-11-08T11:20:56,509 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/.tmp/testFamily/32fc1396e26f4ac2a1da022ea2a8cd70 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/testFamily/32fc1396e26f4ac2a1da022ea2a8cd70 2024-11-08T11:20:56,509 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/table/5baa896784ef4cf9a56f45a0bc36ea3e, entries=58, sequenceid=149, filesize=9.6 K 2024-11-08T11:20:56,510 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 956ms, sequenceid=149, compaction requested=false 2024-11-08T11:20:56,515 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-08T11:20:56,516 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/testFamily/32fc1396e26f4ac2a1da022ea2a8cd70, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:56,516 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T11:20:56,516 INFO [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T11:20:56,516 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T11:20:56,516 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8854a0dae2dc4ae8fc413fa10c0ab93c in 433ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:56,517 DEBUG [RS_CLOSE_META-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T11:20:56,520 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/8854a0dae2dc4ae8fc413fa10c0ab93c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:56,521 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:20:56,521 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8854a0dae2dc4ae8fc413fa10c0ab93c: 2024-11-08T11:20:56,521 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731064717953.8854a0dae2dc4ae8fc413fa10c0ab93c. 2024-11-08T11:20:56,521 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c1a5df7938043bbc4d0adc89c260dff6, disabling compactions & flushes 2024-11-08T11:20:56,521 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:56,521 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:56,521 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. after waiting 0 ms 2024-11-08T11:20:56,521 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:56,521 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c1a5df7938043bbc4d0adc89c260dff6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:56,544 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/.tmp/testFamily/5d000f0361b14678ac4280b1f747fbe2 is 61, key is row-0/testFamily:testQualifier/1731064840465/Put/seqid=0 2024-11-08T11:20:56,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741897_1073 (size=5049) 2024-11-08T11:20:56,556 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-08T11:20:56,556 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:56,756 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:56,949 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/.tmp/testFamily/5d000f0361b14678ac4280b1f747fbe2 2024-11-08T11:20:56,957 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, c1a5df7938043bbc4d0adc89c260dff6, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:56,957 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/.tmp/testFamily/5d000f0361b14678ac4280b1f747fbe2 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/testFamily/5d000f0361b14678ac4280b1f747fbe2 2024-11-08T11:20:56,964 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/testFamily/5d000f0361b14678ac4280b1f747fbe2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:56,965 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c1a5df7938043bbc4d0adc89c260dff6 in 444ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:56,969 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_1_-true/c1a5df7938043bbc4d0adc89c260dff6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:56,969 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:56,970 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c1a5df7938043bbc4d0adc89c260dff6: 2024-11-08T11:20:56,970 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731064837448.c1a5df7938043bbc4d0adc89c260dff6. 2024-11-08T11:20:56,970 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a198a6c470d9a189e91f0680bf15f8f4, disabling compactions & flushes 2024-11-08T11:20:56,970 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:20:56,970 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:20:56,970 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. after waiting 0 ms 2024-11-08T11:20:56,970 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:20:56,970 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a198a6c470d9a189e91f0680bf15f8f4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:56,985 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/.tmp/testFamily/2fe7ef23256b4b9b9f590017f6afa80a is 61, key is row-0/testFamily:testQualifier/1731064668120/Put/seqid=0 2024-11-08T11:20:56,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741898_1074 (size=5049) 2024-11-08T11:20:57,157 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:57,198 ERROR [LeaseRenewer:jenkins@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46645,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:20:57,357 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a198a6c470d9a189e91f0680bf15f8f4, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, decbab74a7be3e3a5b25bee2e10ab413, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:57,390 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/.tmp/testFamily/2fe7ef23256b4b9b9f590017f6afa80a 2024-11-08T11:20:57,398 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/.tmp/testFamily/2fe7ef23256b4b9b9f590017f6afa80a as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/testFamily/2fe7ef23256b4b9b9f590017f6afa80a 2024-11-08T11:20:57,405 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/testFamily/2fe7ef23256b4b9b9f590017f6afa80a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:57,406 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a198a6c470d9a189e91f0680bf15f8f4 in 436ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:57,410 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/a198a6c470d9a189e91f0680bf15f8f4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:57,410 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:20:57,410 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a198a6c470d9a189e91f0680bf15f8f4: 2024-11-08T11:20:57,410 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731064667074.a198a6c470d9a189e91f0680bf15f8f4. 2024-11-08T11:20:57,411 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing decbab74a7be3e3a5b25bee2e10ab413, disabling compactions & flushes 2024-11-08T11:20:57,411 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:20:57,411 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:20:57,411 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. after waiting 0 ms 2024-11-08T11:20:57,411 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:20:57,411 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing decbab74a7be3e3a5b25bee2e10ab413 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:57,426 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/.tmp/testFamily/b038395dedb649c0a68063951891c944 is 61, key is row-0/testFamily:testQualifier/1731064693372/Put/seqid=0 2024-11-08T11:20:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741899_1075 (size=5049) 2024-11-08T11:20:57,431 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/.tmp/testFamily/b038395dedb649c0a68063951891c944 2024-11-08T11:20:57,438 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/.tmp/testFamily/b038395dedb649c0a68063951891c944 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/testFamily/b038395dedb649c0a68063951891c944 2024-11-08T11:20:57,445 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/testFamily/b038395dedb649c0a68063951891c944, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:57,445 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for decbab74a7be3e3a5b25bee2e10ab413 in 34ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:57,450 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/decbab74a7be3e3a5b25bee2e10ab413/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:57,451 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:20:57,451 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for decbab74a7be3e3a5b25bee2e10ab413: 2024-11-08T11:20:57,451 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731064690351.decbab74a7be3e3a5b25bee2e10ab413. 2024-11-08T11:20:57,451 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 47efa016b96d371d1d5a39cfe0614766, disabling compactions & flushes 2024-11-08T11:20:57,451 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:20:57,451 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:20:57,451 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. after waiting 0 ms 2024-11-08T11:20:57,452 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:20:57,452 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 47efa016b96d371d1d5a39cfe0614766 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:57,467 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/.tmp/testFamily/79a2bc1886bf4db08494f84f36bda7e1 is 61, key is row-0/testFamily:testQualifier/1731064714290/Put/seqid=0 2024-11-08T11:20:57,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741900_1076 (size=5049) 2024-11-08T11:20:57,557 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 24 regions to close 2024-11-08T11:20:57,558 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:57,758 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 47efa016b96d371d1d5a39cfe0614766, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:57,873 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/.tmp/testFamily/79a2bc1886bf4db08494f84f36bda7e1 2024-11-08T11:20:57,880 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/.tmp/testFamily/79a2bc1886bf4db08494f84f36bda7e1 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/testFamily/79a2bc1886bf4db08494f84f36bda7e1 2024-11-08T11:20:57,887 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/testFamily/79a2bc1886bf4db08494f84f36bda7e1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:57,888 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 47efa016b96d371d1d5a39cfe0614766 in 436ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:57,892 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_0_-false/47efa016b96d371d1d5a39cfe0614766/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:57,893 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:20:57,893 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 47efa016b96d371d1d5a39cfe0614766: 2024-11-08T11:20:57,893 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731064711264.47efa016b96d371d1d5a39cfe0614766. 2024-11-08T11:20:57,894 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1ba55c509ca5439d425a248d570199f6, disabling compactions & flushes 2024-11-08T11:20:57,894 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:20:57,894 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:20:57,894 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. after waiting 0 ms 2024-11-08T11:20:57,894 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:20:57,894 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1ba55c509ca5439d425a248d570199f6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:57,912 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/.tmp/testFamily/bd4ccd23c3014e5f82989318157ed925 is 61, key is row-0/testFamily:testQualifier/1731064799872/Put/seqid=0 2024-11-08T11:20:57,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741901_1077 (size=5049) 2024-11-08T11:20:57,958 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:57,961 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46645 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46645,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T11:20:58,158 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 1ba55c509ca5439d425a248d570199f6, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:58,322 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/.tmp/testFamily/bd4ccd23c3014e5f82989318157ed925 2024-11-08T11:20:58,330 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/.tmp/testFamily/bd4ccd23c3014e5f82989318157ed925 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/testFamily/bd4ccd23c3014e5f82989318157ed925 2024-11-08T11:20:58,337 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/testFamily/bd4ccd23c3014e5f82989318157ed925, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:58,338 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1ba55c509ca5439d425a248d570199f6 in 444ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:58,342 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_1_-true/1ba55c509ca5439d425a248d570199f6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:58,343 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:20:58,343 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1ba55c509ca5439d425a248d570199f6: 2024-11-08T11:20:58,343 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731064798852.1ba55c509ca5439d425a248d570199f6. 2024-11-08T11:20:58,344 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3ecc4bd45e63090de8e9f1621e42d4d0, disabling compactions & flushes 2024-11-08T11:20:58,344 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:58,344 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:58,344 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. after waiting 0 ms 2024-11-08T11:20:58,344 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:58,344 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3ecc4bd45e63090de8e9f1621e42d4d0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:58,358 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:58,367 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/.tmp/testFamily/271e0d61ba414dedb26f4a4d0d40fcc4 is 61, key is row-0/testFamily:testQualifier/1731064820093/Put/seqid=0 2024-11-08T11:20:58,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741902_1078 (size=5049) 2024-11-08T11:20:58,559 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 22 regions to close 2024-11-08T11:20:58,559 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:58,759 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 3ecc4bd45e63090de8e9f1621e42d4d0, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:58,773 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/.tmp/testFamily/271e0d61ba414dedb26f4a4d0d40fcc4 2024-11-08T11:20:58,781 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/.tmp/testFamily/271e0d61ba414dedb26f4a4d0d40fcc4 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/testFamily/271e0d61ba414dedb26f4a4d0d40fcc4 2024-11-08T11:20:58,788 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/testFamily/271e0d61ba414dedb26f4a4d0d40fcc4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:58,789 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3ecc4bd45e63090de8e9f1621e42d4d0 in 445ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:58,794 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/3ecc4bd45e63090de8e9f1621e42d4d0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:58,795 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:58,795 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3ecc4bd45e63090de8e9f1621e42d4d0: 2024-11-08T11:20:58,795 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731064817077.3ecc4bd45e63090de8e9f1621e42d4d0. 2024-11-08T11:20:58,795 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 28a413aad77fe00c80109f757f9eff1d, disabling compactions & flushes 2024-11-08T11:20:58,795 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:20:58,795 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:20:58,795 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. after waiting 0 ms 2024-11-08T11:20:58,795 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:20:58,795 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 28a413aad77fe00c80109f757f9eff1d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:58,812 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/.tmp/testFamily/aa77d4733b7742f189870dd6da70ea26 is 61, key is row-0/testFamily:testQualifier/1731064716870/Put/seqid=0 2024-11-08T11:20:58,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741903_1079 (size=5049) 2024-11-08T11:20:58,959 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:59,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T11:20:59,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T11:20:59,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T11:20:59,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T11:20:59,160 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 28a413aad77fe00c80109f757f9eff1d, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:59,218 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/.tmp/testFamily/aa77d4733b7742f189870dd6da70ea26 2024-11-08T11:20:59,225 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/.tmp/testFamily/aa77d4733b7742f189870dd6da70ea26 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/testFamily/aa77d4733b7742f189870dd6da70ea26 2024-11-08T11:20:59,231 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/testFamily/aa77d4733b7742f189870dd6da70ea26, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:59,232 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 28a413aad77fe00c80109f757f9eff1d in 437ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:59,237 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNext_0_-false/28a413aad77fe00c80109f757f9eff1d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:59,237 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:20:59,237 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 28a413aad77fe00c80109f757f9eff1d: 2024-11-08T11:20:59,237 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731064715851.28a413aad77fe00c80109f757f9eff1d. 2024-11-08T11:20:59,238 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 810cb95db16ea2b3418f12a23ced7304, disabling compactions & flushes 2024-11-08T11:20:59,238 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:59,238 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:59,238 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. after waiting 0 ms 2024-11-08T11:20:59,238 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:59,238 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 810cb95db16ea2b3418f12a23ced7304 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:59,254 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/.tmp/testFamily/c82482c532ae4a49937b1086619a5feb is 61, key is row-0/testFamily:testQualifier/1731064783541/Put/seqid=0 2024-11-08T11:20:59,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741904_1080 (size=5049) 2024-11-08T11:20:59,309 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4132839332912300167 expired clientIPAndPort=172.17.0.2:58614, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:20:59,360 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:59,560 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 20 regions to close 2024-11-08T11:20:59,560 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 810cb95db16ea2b3418f12a23ced7304, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:59,659 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/.tmp/testFamily/c82482c532ae4a49937b1086619a5feb 2024-11-08T11:20:59,667 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/.tmp/testFamily/c82482c532ae4a49937b1086619a5feb as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/testFamily/c82482c532ae4a49937b1086619a5feb 2024-11-08T11:20:59,674 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/testFamily/c82482c532ae4a49937b1086619a5feb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:20:59,675 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 810cb95db16ea2b3418f12a23ced7304 in 437ms, sequenceid=8, compaction requested=false 2024-11-08T11:20:59,679 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_1_-true/810cb95db16ea2b3418f12a23ced7304/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:20:59,680 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:59,680 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 810cb95db16ea2b3418f12a23ced7304: 2024-11-08T11:20:59,680 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731064780522.810cb95db16ea2b3418f12a23ced7304. 2024-11-08T11:20:59,680 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 27c4e829c2cf184f5508de640a25b0bb, disabling compactions & flushes 2024-11-08T11:20:59,680 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:20:59,680 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:20:59,680 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. after waiting 0 ms 2024-11-08T11:20:59,680 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:20:59,680 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 27c4e829c2cf184f5508de640a25b0bb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:20:59,696 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/.tmp/testFamily/5ab53b5b65b742519f56f6aaa5460432 is 61, key is row-0/testFamily:testQualifier/1731064678212/Put/seqid=0 2024-11-08T11:20:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741905_1081 (size=5049) 2024-11-08T11:20:59,760 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:20:59,961 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 27c4e829c2cf184f5508de640a25b0bb, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:00,101 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/.tmp/testFamily/5ab53b5b65b742519f56f6aaa5460432 2024-11-08T11:21:00,108 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/.tmp/testFamily/5ab53b5b65b742519f56f6aaa5460432 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/testFamily/5ab53b5b65b742519f56f6aaa5460432 2024-11-08T11:21:00,114 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/testFamily/5ab53b5b65b742519f56f6aaa5460432, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:00,115 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 27c4e829c2cf184f5508de640a25b0bb in 435ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:00,119 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/27c4e829c2cf184f5508de640a25b0bb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:00,120 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:21:00,120 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 27c4e829c2cf184f5508de640a25b0bb: 2024-11-08T11:21:00,120 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731064677189.27c4e829c2cf184f5508de640a25b0bb. 2024-11-08T11:21:00,120 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 04d2e8f0a8c6ed33efc2158fe9bd9408, disabling compactions & flushes 2024-11-08T11:21:00,120 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:21:00,120 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:21:00,120 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. after waiting 0 ms 2024-11-08T11:21:00,120 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:21:00,120 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 04d2e8f0a8c6ed33efc2158fe9bd9408 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:00,136 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/.tmp/testFamily/69321d5b3727481d9527b2a95a2df4d3 is 61, key is row-0/testFamily:testQualifier/1731064809438/Put/seqid=0 2024-11-08T11:21:00,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741906_1082 (size=5049) 2024-11-08T11:21:00,161 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:00,361 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 04d2e8f0a8c6ed33efc2158fe9bd9408, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:00,541 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/.tmp/testFamily/69321d5b3727481d9527b2a95a2df4d3 2024-11-08T11:21:00,548 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/.tmp/testFamily/69321d5b3727481d9527b2a95a2df4d3 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/testFamily/69321d5b3727481d9527b2a95a2df4d3 2024-11-08T11:21:00,554 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/testFamily/69321d5b3727481d9527b2a95a2df4d3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:00,555 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 04d2e8f0a8c6ed33efc2158fe9bd9408 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:00,559 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnClose_1_-true/04d2e8f0a8c6ed33efc2158fe9bd9408/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:00,560 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:21:00,560 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 04d2e8f0a8c6ed33efc2158fe9bd9408: 2024-11-08T11:21:00,560 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731064806417.04d2e8f0a8c6ed33efc2158fe9bd9408. 2024-11-08T11:21:00,560 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8c25b44c431bf3f90b6b41bc98a7e29d, disabling compactions & flushes 2024-11-08T11:21:00,560 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:21:00,560 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:21:00,560 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. after waiting 0 ms 2024-11-08T11:21:00,560 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:21:00,560 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8c25b44c431bf3f90b6b41bc98a7e29d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:00,561 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 17 regions to close 2024-11-08T11:21:00,561 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:00,577 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/.tmp/testFamily/d90bb58dacc64646b91d7d8934ed1fa5 is 61, key is row-0/testFamily:testQualifier/1731064690291/Put/seqid=0 2024-11-08T11:21:00,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741907_1083 (size=5049) 2024-11-08T11:21:00,762 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:00,962 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 8c25b44c431bf3f90b6b41bc98a7e29d, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:00,982 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/.tmp/testFamily/d90bb58dacc64646b91d7d8934ed1fa5 2024-11-08T11:21:00,990 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/.tmp/testFamily/d90bb58dacc64646b91d7d8934ed1fa5 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/testFamily/d90bb58dacc64646b91d7d8934ed1fa5 2024-11-08T11:21:00,997 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/testFamily/d90bb58dacc64646b91d7d8934ed1fa5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:00,998 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8c25b44c431bf3f90b6b41bc98a7e29d in 438ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:01,002 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextException_0_-false/8c25b44c431bf3f90b6b41bc98a7e29d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:01,003 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:21:01,003 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8c25b44c431bf3f90b6b41bc98a7e29d: 2024-11-08T11:21:01,003 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731064687269.8c25b44c431bf3f90b6b41bc98a7e29d. 2024-11-08T11:21:01,003 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8029e11915c8ac0360de37aa6659111e, disabling compactions & flushes 2024-11-08T11:21:01,003 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:21:01,003 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:21:01,003 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. after waiting 0 ms 2024-11-08T11:21:01,003 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:21:01,003 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8029e11915c8ac0360de37aa6659111e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:01,019 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/.tmp/testFamily/0eacb8d827ec4ef2824386f8d13b3106 is 61, key is row-0/testFamily:testQualifier/1731064731047/Put/seqid=0 2024-11-08T11:21:01,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741908_1084 (size=5049) 2024-11-08T11:21:01,162 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:01,362 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 8029e11915c8ac0360de37aa6659111e, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:01,424 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/.tmp/testFamily/0eacb8d827ec4ef2824386f8d13b3106 2024-11-08T11:21:01,431 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/.tmp/testFamily/0eacb8d827ec4ef2824386f8d13b3106 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/testFamily/0eacb8d827ec4ef2824386f8d13b3106 2024-11-08T11:21:01,438 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/testFamily/0eacb8d827ec4ef2824386f8d13b3106, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:01,438 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8029e11915c8ac0360de37aa6659111e in 435ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:01,443 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/8029e11915c8ac0360de37aa6659111e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:01,444 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:21:01,444 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8029e11915c8ac0360de37aa6659111e: 2024-11-08T11:21:01,444 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731064728027.8029e11915c8ac0360de37aa6659111e. 2024-11-08T11:21:01,444 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ce6c8c461db85938ff504cfbc5c00973, disabling compactions & flushes 2024-11-08T11:21:01,445 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:21:01,445 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:21:01,445 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. after waiting 0 ms 2024-11-08T11:21:01,445 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:21:01,445 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ce6c8c461db85938ff504cfbc5c00973 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:01,460 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/.tmp/testFamily/3e1d69616285477fa9526beaf6339e92 is 61, key is row-0/testFamily:testQualifier/1731064752253/Put/seqid=0 2024-11-08T11:21:01,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741909_1085 (size=5049) 2024-11-08T11:21:01,563 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 15 regions to close 2024-11-08T11:21:01,563 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:01,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:21:01,763 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, ce6c8c461db85938ff504cfbc5c00973, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:01,865 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/.tmp/testFamily/3e1d69616285477fa9526beaf6339e92 2024-11-08T11:21:01,872 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/.tmp/testFamily/3e1d69616285477fa9526beaf6339e92 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/testFamily/3e1d69616285477fa9526beaf6339e92 2024-11-08T11:21:01,879 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/testFamily/3e1d69616285477fa9526beaf6339e92, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:01,880 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ce6c8c461db85938ff504cfbc5c00973 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:01,884 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ce6c8c461db85938ff504cfbc5c00973/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:01,885 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:21:01,885 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ce6c8c461db85938ff504cfbc5c00973: 2024-11-08T11:21:01,885 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731064749232.ce6c8c461db85938ff504cfbc5c00973. 2024-11-08T11:21:01,885 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b96aee56c445cd36471b3c7201edc90d, disabling compactions & flushes 2024-11-08T11:21:01,885 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:21:01,885 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:21:01,885 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. after waiting 0 ms 2024-11-08T11:21:01,885 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:21:01,885 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b96aee56c445cd36471b3c7201edc90d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:01,901 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/.tmp/testFamily/b49ed9e103014e789e53aa63b6531952 is 61, key is row-0/testFamily:testQualifier/1731064758314/Put/seqid=0 2024-11-08T11:21:01,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741910_1086 (size=5049) 2024-11-08T11:21:01,963 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:02,163 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, b96aee56c445cd36471b3c7201edc90d, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:02,306 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/.tmp/testFamily/b49ed9e103014e789e53aa63b6531952 2024-11-08T11:21:02,313 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/.tmp/testFamily/b49ed9e103014e789e53aa63b6531952 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/testFamily/b49ed9e103014e789e53aa63b6531952 2024-11-08T11:21:02,319 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/testFamily/b49ed9e103014e789e53aa63b6531952, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:02,320 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b96aee56c445cd36471b3c7201edc90d in 435ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:02,324 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/b96aee56c445cd36471b3c7201edc90d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:02,325 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:21:02,325 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b96aee56c445cd36471b3c7201edc90d: 2024-11-08T11:21:02,325 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731064755296.b96aee56c445cd36471b3c7201edc90d. 2024-11-08T11:21:02,325 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f2b790a2329bc0309d783fbb87b96ddd, disabling compactions & flushes 2024-11-08T11:21:02,325 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:21:02,325 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:21:02,325 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. after waiting 0 ms 2024-11-08T11:21:02,325 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:21:02,325 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f2b790a2329bc0309d783fbb87b96ddd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:02,342 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/.tmp/testFamily/9c0c05b208104871b8a76e6ac824be46 is 61, key is row-0/testFamily:testQualifier/1731064852523/Put/seqid=0 2024-11-08T11:21:02,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741911_1087 (size=5049) 2024-11-08T11:21:02,364 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:02,564 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 13 regions to close 2024-11-08T11:21:02,564 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:02,748 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/.tmp/testFamily/9c0c05b208104871b8a76e6ac824be46 2024-11-08T11:21:02,756 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/.tmp/testFamily/9c0c05b208104871b8a76e6ac824be46 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/testFamily/9c0c05b208104871b8a76e6ac824be46 2024-11-08T11:21:02,762 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/testFamily/9c0c05b208104871b8a76e6ac824be46, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:02,762 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f2b790a2329bc0309d783fbb87b96ddd in 437ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:02,764 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, f2b790a2329bc0309d783fbb87b96ddd, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:02,767 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/f2b790a2329bc0309d783fbb87b96ddd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:02,768 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:21:02,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f2b790a2329bc0309d783fbb87b96ddd: 2024-11-08T11:21:02,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731064849507.f2b790a2329bc0309d783fbb87b96ddd. 2024-11-08T11:21:02,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b6492f1f8d3b65f481a71f69b3f63ffa, disabling compactions & flushes 2024-11-08T11:21:02,768 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:21:02,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:21:02,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. after waiting 0 ms 2024-11-08T11:21:02,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:21:02,768 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b6492f1f8d3b65f481a71f69b3f63ffa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:02,785 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/.tmp/testFamily/1e7fb77d0df341459af63e7fd33f6d08 is 61, key is row-0/testFamily:testQualifier/1731064761383/Put/seqid=0 2024-11-08T11:21:02,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741912_1088 (size=5049) 2024-11-08T11:21:02,964 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:03,165 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, b6492f1f8d3b65f481a71f69b3f63ffa, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:03,190 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/.tmp/testFamily/1e7fb77d0df341459af63e7fd33f6d08 2024-11-08T11:21:03,199 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/.tmp/testFamily/1e7fb77d0df341459af63e7fd33f6d08 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/testFamily/1e7fb77d0df341459af63e7fd33f6d08 2024-11-08T11:21:03,206 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/testFamily/1e7fb77d0df341459af63e7fd33f6d08, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:03,207 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b6492f1f8d3b65f481a71f69b3f63ffa in 438ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:03,211 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/b6492f1f8d3b65f481a71f69b3f63ffa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:03,211 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:21:03,211 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b6492f1f8d3b65f481a71f69b3f63ffa: 2024-11-08T11:21:03,212 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731064758366.b6492f1f8d3b65f481a71f69b3f63ffa. 2024-11-08T11:21:03,212 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e187798f3f2dc14ef76d302202186f7c, disabling compactions & flushes 2024-11-08T11:21:03,212 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:21:03,212 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:21:03,212 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. after waiting 0 ms 2024-11-08T11:21:03,212 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:21:03,212 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e187798f3f2dc14ef76d302202186f7c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:03,227 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/.tmp/testFamily/56145f24ee22429682b4423dcd267f82 is 61, key is row-0/testFamily:testQualifier/1731064696468/Put/seqid=0 2024-11-08T11:21:03,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741913_1089 (size=5049) 2024-11-08T11:21:03,365 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:03,565 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 11 regions to close 2024-11-08T11:21:03,565 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, da78e98c858db35fcecc75c6fe00ecc8, e187798f3f2dc14ef76d302202186f7c, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:03,633 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/.tmp/testFamily/56145f24ee22429682b4423dcd267f82 2024-11-08T11:21:03,641 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/.tmp/testFamily/56145f24ee22429682b4423dcd267f82 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/testFamily/56145f24ee22429682b4423dcd267f82 2024-11-08T11:21:03,647 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/testFamily/56145f24ee22429682b4423dcd267f82, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:03,649 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e187798f3f2dc14ef76d302202186f7c in 437ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:03,655 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/e187798f3f2dc14ef76d302202186f7c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:03,656 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:21:03,656 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e187798f3f2dc14ef76d302202186f7c: 2024-11-08T11:21:03,656 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731064695442.e187798f3f2dc14ef76d302202186f7c. 2024-11-08T11:21:03,656 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ad75048110dcede8a9d5403d1e5644c9, disabling compactions & flushes 2024-11-08T11:21:03,656 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:21:03,657 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:21:03,657 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. after waiting 0 ms 2024-11-08T11:21:03,657 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:21:03,657 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ad75048110dcede8a9d5403d1e5644c9 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-08T11:21:03,673 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/.tmp/info/75dce0aaabb24108bece10578888738a is 45, key is default/info:d/1731064663613/Put/seqid=0 2024-11-08T11:21:03,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741914_1090 (size=5037) 2024-11-08T11:21:03,766 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:03,966 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, ad75048110dcede8a9d5403d1e5644c9, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:04,079 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/.tmp/info/75dce0aaabb24108bece10578888738a 2024-11-08T11:21:04,087 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/.tmp/info/75dce0aaabb24108bece10578888738a as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/info/75dce0aaabb24108bece10578888738a 2024-11-08T11:21:04,092 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/info/75dce0aaabb24108bece10578888738a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T11:21:04,093 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for ad75048110dcede8a9d5403d1e5644c9 in 436ms, sequenceid=6, compaction requested=false 2024-11-08T11:21:04,099 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/namespace/ad75048110dcede8a9d5403d1e5644c9/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T11:21:04,099 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:21:04,099 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ad75048110dcede8a9d5403d1e5644c9: 2024-11-08T11:21:04,099 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731064662263.ad75048110dcede8a9d5403d1e5644c9. 2024-11-08T11:21:04,100 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 967fd2a06121897b47c8cd11f2da98b3, disabling compactions & flushes 2024-11-08T11:21:04,100 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:21:04,100 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:21:04,100 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. after waiting 0 ms 2024-11-08T11:21:04,100 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:21:04,100 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 967fd2a06121897b47c8cd11f2da98b3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:04,115 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/.tmp/testFamily/52b0ff119a3e46518111c1fab22d428e is 61, key is row-0/testFamily:testQualifier/1731064828218/Put/seqid=0 2024-11-08T11:21:04,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741915_1091 (size=5049) 2024-11-08T11:21:04,166 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:04,366 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, 967fd2a06121897b47c8cd11f2da98b3, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:04,520 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/.tmp/testFamily/52b0ff119a3e46518111c1fab22d428e 2024-11-08T11:21:04,529 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/.tmp/testFamily/52b0ff119a3e46518111c1fab22d428e as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/testFamily/52b0ff119a3e46518111c1fab22d428e 2024-11-08T11:21:04,535 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/testFamily/52b0ff119a3e46518111c1fab22d428e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:04,536 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 967fd2a06121897b47c8cd11f2da98b3 in 436ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:04,544 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/967fd2a06121897b47c8cd11f2da98b3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:04,544 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:21:04,545 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 967fd2a06121897b47c8cd11f2da98b3: 2024-11-08T11:21:04,545 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731064827184.967fd2a06121897b47c8cd11f2da98b3. 2024-11-08T11:21:04,545 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 805a942bf739e1744fdcc1a8ca9185fe, disabling compactions & flushes 2024-11-08T11:21:04,545 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:21:04,545 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:21:04,545 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. after waiting 0 ms 2024-11-08T11:21:04,545 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:21:04,545 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 805a942bf739e1744fdcc1a8ca9185fe 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:04,567 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 8 regions to close 2024-11-08T11:21:04,567 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1603): Online Regions={805a942bf739e1744fdcc1a8ca9185fe=testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe., da78e98c858db35fcecc75c6fe00ecc8=testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8., 012d9e50c5a5dfe248e1e5d5c6aa397b=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b., 2ae2b67ac99527ab93cd9d51d12df9c3=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3., a97e8684ef3557cc5c8111e42191ba0f=testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f., 0aa76efa23785cc8435444278f71e915=hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915., 7fdf3e5624b7e358e192563c0644e9a1=hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1., fcce6bd67ff4fe800c5856874d71d076=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.} 2024-11-08T11:21:04,567 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:04,567 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/.tmp/testFamily/5d023824c6484ab584e0c1cedc55186d is 61, key is row-0/testFamily:testQualifier/1731064795784/Put/seqid=0 2024-11-08T11:21:04,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741916_1092 (size=5049) 2024-11-08T11:21:04,767 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:04,967 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, 805a942bf739e1744fdcc1a8ca9185fe, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:04,982 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/.tmp/testFamily/5d023824c6484ab584e0c1cedc55186d 2024-11-08T11:21:04,989 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/.tmp/testFamily/5d023824c6484ab584e0c1cedc55186d as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/testFamily/5d023824c6484ab584e0c1cedc55186d 2024-11-08T11:21:04,995 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/testFamily/5d023824c6484ab584e0c1cedc55186d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:04,996 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 805a942bf739e1744fdcc1a8ca9185fe in 451ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:05,000 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/805a942bf739e1744fdcc1a8ca9185fe/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:05,001 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:21:05,001 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 805a942bf739e1744fdcc1a8ca9185fe: 2024-11-08T11:21:05,001 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731064794762.805a942bf739e1744fdcc1a8ca9185fe. 2024-11-08T11:21:05,001 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing da78e98c858db35fcecc75c6fe00ecc8, disabling compactions & flushes 2024-11-08T11:21:05,001 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:21:05,001 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:21:05,001 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. after waiting 0 ms 2024-11-08T11:21:05,001 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:21:05,001 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing da78e98c858db35fcecc75c6fe00ecc8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:05,018 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/.tmp/testFamily/9bf428d655ba4b19a9542c706a97231f is 61, key is row-0/testFamily:testQualifier/1731064786635/Put/seqid=0 2024-11-08T11:21:05,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741917_1093 (size=5049) 2024-11-08T11:21:05,168 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:05,368 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, da78e98c858db35fcecc75c6fe00ecc8, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:05,423 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/.tmp/testFamily/9bf428d655ba4b19a9542c706a97231f 2024-11-08T11:21:05,430 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/.tmp/testFamily/9bf428d655ba4b19a9542c706a97231f as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/testFamily/9bf428d655ba4b19a9542c706a97231f 2024-11-08T11:21:05,435 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/testFamily/9bf428d655ba4b19a9542c706a97231f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:05,436 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for da78e98c858db35fcecc75c6fe00ecc8 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:05,440 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/da78e98c858db35fcecc75c6fe00ecc8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:05,441 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:21:05,441 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for da78e98c858db35fcecc75c6fe00ecc8: 2024-11-08T11:21:05,441 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731064783618.da78e98c858db35fcecc75c6fe00ecc8. 2024-11-08T11:21:05,441 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 012d9e50c5a5dfe248e1e5d5c6aa397b, disabling compactions & flushes 2024-11-08T11:21:05,441 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:21:05,441 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:21:05,441 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. after waiting 0 ms 2024-11-08T11:21:05,441 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:21:05,441 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 012d9e50c5a5dfe248e1e5d5c6aa397b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:05,456 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/.tmp/testFamily/5a1ff1b9ce8244b9b6005d8dcbb74e25 is 61, key is row-0/testFamily:testQualifier/1731064771471/Put/seqid=0 2024-11-08T11:21:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741918_1094 (size=5049) 2024-11-08T11:21:05,568 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 6 regions to close 2024-11-08T11:21:05,568 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1603): Online Regions={012d9e50c5a5dfe248e1e5d5c6aa397b=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b., 2ae2b67ac99527ab93cd9d51d12df9c3=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3., a97e8684ef3557cc5c8111e42191ba0f=testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f., 0aa76efa23785cc8435444278f71e915=hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915., 7fdf3e5624b7e358e192563c0644e9a1=hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1., fcce6bd67ff4fe800c5856874d71d076=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.} 2024-11-08T11:21:05,568 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:05,769 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 012d9e50c5a5dfe248e1e5d5c6aa397b, 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:05,861 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/.tmp/testFamily/5a1ff1b9ce8244b9b6005d8dcbb74e25 2024-11-08T11:21:05,868 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/.tmp/testFamily/5a1ff1b9ce8244b9b6005d8dcbb74e25 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/testFamily/5a1ff1b9ce8244b9b6005d8dcbb74e25 2024-11-08T11:21:05,873 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/testFamily/5a1ff1b9ce8244b9b6005d8dcbb74e25, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:05,874 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 012d9e50c5a5dfe248e1e5d5c6aa397b in 433ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:05,878 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/012d9e50c5a5dfe248e1e5d5c6aa397b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:05,879 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:21:05,879 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 012d9e50c5a5dfe248e1e5d5c6aa397b: 2024-11-08T11:21:05,879 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731064770453.012d9e50c5a5dfe248e1e5d5c6aa397b. 2024-11-08T11:21:05,879 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2ae2b67ac99527ab93cd9d51d12df9c3, disabling compactions & flushes 2024-11-08T11:21:05,879 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:21:05,879 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:21:05,879 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. after waiting 0 ms 2024-11-08T11:21:05,879 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:21:05,879 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2ae2b67ac99527ab93cd9d51d12df9c3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:05,894 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/.tmp/testFamily/27666ccc23ba4c4b8a49a23ff703895c is 61, key is row-0/testFamily:testQualifier/1731064666836/Put/seqid=0 2024-11-08T11:21:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741919_1095 (size=5049) 2024-11-08T11:21:05,969 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:06,169 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 2ae2b67ac99527ab93cd9d51d12df9c3, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:06,299 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/.tmp/testFamily/27666ccc23ba4c4b8a49a23ff703895c 2024-11-08T11:21:06,306 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/.tmp/testFamily/27666ccc23ba4c4b8a49a23ff703895c as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/testFamily/27666ccc23ba4c4b8a49a23ff703895c 2024-11-08T11:21:06,311 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/testFamily/27666ccc23ba4c4b8a49a23ff703895c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:06,312 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ae2b67ac99527ab93cd9d51d12df9c3 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/2ae2b67ac99527ab93cd9d51d12df9c3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:06,316 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2ae2b67ac99527ab93cd9d51d12df9c3: 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731064663787.2ae2b67ac99527ab93cd9d51d12df9c3. 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a97e8684ef3557cc5c8111e42191ba0f, disabling compactions & flushes 2024-11-08T11:21:06,316 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. after waiting 0 ms 2024-11-08T11:21:06,316 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:21:06,317 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a97e8684ef3557cc5c8111e42191ba0f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:06,331 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/.tmp/testFamily/46772d403d4045c39d41854f71db6c9e is 61, key is row-0/testFamily:testQualifier/1731064700548/Put/seqid=0 2024-11-08T11:21:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741920_1096 (size=5049) 2024-11-08T11:21:06,369 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:06,504 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T11:21:06,569 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-11-08T11:21:06,569 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1603): Online Regions={a97e8684ef3557cc5c8111e42191ba0f=testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f., 0aa76efa23785cc8435444278f71e915=hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915., 7fdf3e5624b7e358e192563c0644e9a1=hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1., fcce6bd67ff4fe800c5856874d71d076=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.} 2024-11-08T11:21:06,569 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 7fdf3e5624b7e358e192563c0644e9a1, a97e8684ef3557cc5c8111e42191ba0f, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:06,736 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/.tmp/testFamily/46772d403d4045c39d41854f71db6c9e 2024-11-08T11:21:06,742 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/.tmp/testFamily/46772d403d4045c39d41854f71db6c9e as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/testFamily/46772d403d4045c39d41854f71db6c9e 2024-11-08T11:21:06,748 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/testFamily/46772d403d4045c39d41854f71db6c9e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:06,749 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a97e8684ef3557cc5c8111e42191ba0f in 433ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:06,752 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/a97e8684ef3557cc5c8111e42191ba0f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:06,753 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:21:06,753 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a97e8684ef3557cc5c8111e42191ba0f: 2024-11-08T11:21:06,753 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731064699525.a97e8684ef3557cc5c8111e42191ba0f. 2024-11-08T11:21:06,753 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0aa76efa23785cc8435444278f71e915, disabling compactions & flushes 2024-11-08T11:21:06,753 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:21:06,753 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:21:06,753 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. after waiting 0 ms 2024-11-08T11:21:06,753 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:21:06,753 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0aa76efa23785cc8435444278f71e915 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:06,768 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/.tmp/testFamily/c266827bf4c34c1a86cc540c70aced87 is 61, key is row-0/testFamily:testQualifier/1731064704693/Put/seqid=0 2024-11-08T11:21:06,770 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 7fdf3e5624b7e358e192563c0644e9a1, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:06,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741921_1097 (size=5049) 2024-11-08T11:21:06,970 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 7fdf3e5624b7e358e192563c0644e9a1, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:07,170 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 0aa76efa23785cc8435444278f71e915, 7fdf3e5624b7e358e192563c0644e9a1, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:07,173 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/.tmp/testFamily/c266827bf4c34c1a86cc540c70aced87 2024-11-08T11:21:07,180 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/.tmp/testFamily/c266827bf4c34c1a86cc540c70aced87 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/testFamily/c266827bf4c34c1a86cc540c70aced87 2024-11-08T11:21:07,186 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/testFamily/c266827bf4c34c1a86cc540c70aced87, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:07,187 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0aa76efa23785cc8435444278f71e915 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:07,191 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnClose_0_-false/0aa76efa23785cc8435444278f71e915/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:07,191 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:21:07,191 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0aa76efa23785cc8435444278f71e915: 2024-11-08T11:21:07,191 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731064703648.0aa76efa23785cc8435444278f71e915. 2024-11-08T11:21:07,192 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7fdf3e5624b7e358e192563c0644e9a1, disabling compactions & flushes 2024-11-08T11:21:07,192 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:21:07,192 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:21:07,192 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. after waiting 0 ms 2024-11-08T11:21:07,192 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:21:07,192 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7fdf3e5624b7e358e192563c0644e9a1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:07,207 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/.tmp/testFamily/ddbaf98ecbf547f8879efa71d70e7e39 is 61, key is row-0/testFamily:testQualifier/1731064743131/Put/seqid=0 2024-11-08T11:21:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741922_1098 (size=5049) 2024-11-08T11:21:07,370 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 7fdf3e5624b7e358e192563c0644e9a1, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:07,571 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-08T11:21:07,571 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1603): Online Regions={7fdf3e5624b7e358e192563c0644e9a1=hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1., fcce6bd67ff4fe800c5856874d71d076=testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076.} 2024-11-08T11:21:07,571 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on 7fdf3e5624b7e358e192563c0644e9a1, fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:07,613 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/.tmp/testFamily/ddbaf98ecbf547f8879efa71d70e7e39 2024-11-08T11:21:07,619 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/.tmp/testFamily/ddbaf98ecbf547f8879efa71d70e7e39 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/testFamily/ddbaf98ecbf547f8879efa71d70e7e39 2024-11-08T11:21:07,625 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/testFamily/ddbaf98ecbf547f8879efa71d70e7e39, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:07,626 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7fdf3e5624b7e358e192563c0644e9a1 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:07,630 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/hbase/testMetaScanTimeoutOnNext_0_-false/7fdf3e5624b7e358e192563c0644e9a1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:07,630 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:21:07,630 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7fdf3e5624b7e358e192563c0644e9a1: 2024-11-08T11:21:07,631 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731064740108.7fdf3e5624b7e358e192563c0644e9a1. 2024-11-08T11:21:07,631 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing fcce6bd67ff4fe800c5856874d71d076, disabling compactions & flushes 2024-11-08T11:21:07,631 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:21:07,631 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:21:07,631 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. after waiting 0 ms 2024-11-08T11:21:07,631 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:21:07,631 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing fcce6bd67ff4fe800c5856874d71d076 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T11:21:07,646 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/.tmp/testFamily/a5414496a83540a39d478593f3d97db5 is 61, key is row-0/testFamily:testQualifier/1731064791704/Put/seqid=0 2024-11-08T11:21:07,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741923_1099 (size=5049) 2024-11-08T11:21:07,771 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:07,772 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T11:21:07,971 DEBUG [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1629): Waiting on fcce6bd67ff4fe800c5856874d71d076 2024-11-08T11:21:08,051 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/.tmp/testFamily/a5414496a83540a39d478593f3d97db5 2024-11-08T11:21:08,057 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/.tmp/testFamily/a5414496a83540a39d478593f3d97db5 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/testFamily/a5414496a83540a39d478593f3d97db5 2024-11-08T11:21:08,062 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/testFamily/a5414496a83540a39d478593f3d97db5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T11:21:08,063 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fcce6bd67ff4fe800c5856874d71d076 in 432ms, sequenceid=8, compaction requested=false 2024-11-08T11:21:08,067 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/fcce6bd67ff4fe800c5856874d71d076/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T11:21:08,068 INFO [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:21:08,068 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for fcce6bd67ff4fe800c5856874d71d076: 2024-11-08T11:21:08,068 DEBUG [RS_CLOSE_REGION-regionserver/ae52d81aa32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731064788683.fcce6bd67ff4fe800c5856874d71d076. 2024-11-08T11:21:08,171 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1250): stopping server ae52d81aa32e,33717,1731064629394; all regions closed. 2024-11-08T11:21:08,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741834_1010 (size=56731) 2024-11-08T11:21:08,181 DEBUG [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/oldWALs 2024-11-08T11:21:08,182 INFO [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL ae52d81aa32e%2C33717%2C1731064629394.meta:.meta(num 1731064662020) 2024-11-08T11:21:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741833_1009 (size=60562) 2024-11-08T11:21:08,186 DEBUG [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/oldWALs 2024-11-08T11:21:08,186 INFO [RS:0;ae52d81aa32e:33717 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL ae52d81aa32e%2C33717%2C1731064629394:(num 1731064661650) 2024-11-08T11:21:08,186 DEBUG [RS:0;ae52d81aa32e:33717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:21:08,186 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T11:21:08,186 INFO [RS:0;ae52d81aa32e:33717 {}] hbase.ChoreService(370): Chore service for: regionserver/ae52d81aa32e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-11-08T11:21:08,187 INFO [regionserver/ae52d81aa32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T11:21:08,187 INFO [RS:0;ae52d81aa32e:33717 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:33717 2024-11-08T11:21:08,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae52d81aa32e,33717,1731064629394 2024-11-08T11:21:08,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T11:21:08,194 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae52d81aa32e,33717,1731064629394] 2024-11-08T11:21:08,194 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ae52d81aa32e,33717,1731064629394; numProcessing=1 2024-11-08T11:21:08,195 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ae52d81aa32e,33717,1731064629394 already deleted, retry=false 2024-11-08T11:21:08,195 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ae52d81aa32e,33717,1731064629394 expired; onlineServers=0 2024-11-08T11:21:08,195 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ae52d81aa32e,36127,1731064628628' ***** 2024-11-08T11:21:08,195 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T11:21:08,195 DEBUG [M:0;ae52d81aa32e:36127 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4f3384, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae52d81aa32e/172.17.0.2:0 2024-11-08T11:21:08,195 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegionServer(1224): stopping server ae52d81aa32e,36127,1731064628628 2024-11-08T11:21:08,195 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegionServer(1250): stopping server ae52d81aa32e,36127,1731064628628; all regions closed. 2024-11-08T11:21:08,195 DEBUG [M:0;ae52d81aa32e:36127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T11:21:08,195 DEBUG [M:0;ae52d81aa32e:36127 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T11:21:08,195 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T11:21:08,195 DEBUG [M:0;ae52d81aa32e:36127 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T11:21:08,195 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster-HFileCleaner.large.0-1731064631433 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae52d81aa32e:0:becomeActiveMaster-HFileCleaner.large.0-1731064631433,5,FailOnTimeoutGroup] 2024-11-08T11:21:08,195 DEBUG [master/ae52d81aa32e:0:becomeActiveMaster-HFileCleaner.small.0-1731064631433 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae52d81aa32e:0:becomeActiveMaster-HFileCleaner.small.0-1731064631433,5,FailOnTimeoutGroup] 2024-11-08T11:21:08,196 INFO [M:0;ae52d81aa32e:36127 {}] hbase.ChoreService(370): Chore service for: master/ae52d81aa32e:0 had [] on shutdown 2024-11-08T11:21:08,196 DEBUG [M:0;ae52d81aa32e:36127 {}] master.HMaster(1733): Stopping service threads 2024-11-08T11:21:08,196 INFO [M:0;ae52d81aa32e:36127 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T11:21:08,196 INFO [M:0;ae52d81aa32e:36127 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T11:21:08,197 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T11:21:08,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T11:21:08,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T11:21:08,197 DEBUG [M:0;ae52d81aa32e:36127 {}] zookeeper.ZKUtil(347): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T11:21:08,197 WARN [M:0;ae52d81aa32e:36127 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T11:21:08,197 INFO [M:0;ae52d81aa32e:36127 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-08T11:21:08,197 INFO [M:0;ae52d81aa32e:36127 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T11:21:08,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T11:21:08,197 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T11:21:08,197 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:21:08,197 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:21:08,198 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T11:21:08,198 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:21:08,198 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.21 KB heapSize=519.64 KB 2024-11-08T11:21:08,215 DEBUG [M:0;ae52d81aa32e:36127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1a62468c5ee4434bfd72e4a9e6d9aff is 82, key is hbase:meta,,1/info:regioninfo/1731064662159/Put/seqid=0 2024-11-08T11:21:08,216 INFO [regionserver/ae52d81aa32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T11:21:08,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741924_1100 (size=5672) 2024-11-08T11:21:08,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T11:21:08,294 INFO [RS:0;ae52d81aa32e:33717 {}] regionserver.HRegionServer(1307): Exiting; stopping=ae52d81aa32e,33717,1731064629394; zookeeper connection closed. 2024-11-08T11:21:08,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x101879977df0001, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T11:21:08,294 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c2c54ba {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c2c54ba 2024-11-08T11:21:08,294 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T11:21:08,620 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1a62468c5ee4434bfd72e4a9e6d9aff 2024-11-08T11:21:08,649 DEBUG [M:0;ae52d81aa32e:36127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7bcea04d1f147919598139689a3233a is 829, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731064665016/Put/seqid=0 2024-11-08T11:21:08,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741925_1101 (size=30264) 2024-11-08T11:21:09,055 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=447.65 KB at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7bcea04d1f147919598139689a3233a 2024-11-08T11:21:09,060 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7bcea04d1f147919598139689a3233a 2024-11-08T11:21:09,079 DEBUG [M:0;ae52d81aa32e:36127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84738278474e499e86d0dd6c8d003790 is 69, key is ae52d81aa32e,33717,1731064629394/rs:state/1731064631518/Put/seqid=0 2024-11-08T11:21:09,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741926_1102 (size=5156) 2024-11-08T11:21:09,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T11:21:09,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T11:21:09,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T11:21:09,484 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84738278474e499e86d0dd6c8d003790 2024-11-08T11:21:09,490 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1a62468c5ee4434bfd72e4a9e6d9aff as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c1a62468c5ee4434bfd72e4a9e6d9aff 2024-11-08T11:21:09,495 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c1a62468c5ee4434bfd72e4a9e6d9aff, entries=8, sequenceid=830, filesize=5.5 K 2024-11-08T11:21:09,496 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7bcea04d1f147919598139689a3233a as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7bcea04d1f147919598139689a3233a 2024-11-08T11:21:09,501 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7bcea04d1f147919598139689a3233a 2024-11-08T11:21:09,501 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7bcea04d1f147919598139689a3233a, entries=92, sequenceid=830, filesize=29.6 K 2024-11-08T11:21:09,501 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84738278474e499e86d0dd6c8d003790 as hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84738278474e499e86d0dd6c8d003790 2024-11-08T11:21:09,506 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46645/user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84738278474e499e86d0dd6c8d003790, entries=1, sequenceid=830, filesize=5.0 K 2024-11-08T11:21:09,507 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.21 KB/458964, heapSize ~519.34 KB/531808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=830, compaction requested=false 2024-11-08T11:21:09,508 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T11:21:09,508 DEBUG [M:0;ae52d81aa32e:36127 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T11:21:09,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40011 is added to blk_1073741830_1006 (size=513833) 2024-11-08T11:21:09,511 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/0a97a23e-33a2-2e03-1080-86006c12524a/MasterData/WALs/ae52d81aa32e,36127,1731064628628/ae52d81aa32e%2C36127%2C1731064628628.1731064630952 not finished, retry = 0 2024-11-08T11:21:09,612 INFO [M:0;ae52d81aa32e:36127 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-08T11:21:09,612 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T11:21:09,612 INFO [M:0;ae52d81aa32e:36127 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36127 2024-11-08T11:21:09,615 DEBUG [M:0;ae52d81aa32e:36127 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ae52d81aa32e,36127,1731064628628 already deleted, retry=false 2024-11-08T11:21:09,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T11:21:09,717 INFO [M:0;ae52d81aa32e:36127 {}] regionserver.HRegionServer(1307): Exiting; stopping=ae52d81aa32e,36127,1731064628628; zookeeper connection closed. 2024-11-08T11:21:09,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36127-0x101879977df0000, quorum=127.0.0.1:55518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T11:21:09,722 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@183020ad{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T11:21:09,724 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T11:21:09,724 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T11:21:09,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T11:21:09,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/hadoop.log.dir/,STOPPED} 2024-11-08T11:21:09,727 WARN [BP-503676101-172.17.0.2-1731064625816 heartbeating to localhost/127.0.0.1:46645 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T11:21:09,727 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T11:21:09,727 WARN [BP-503676101-172.17.0.2-1731064625816 heartbeating to localhost/127.0.0.1:46645 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-503676101-172.17.0.2-1731064625816 (Datanode Uuid 309c3c92-024d-4753-b0a4-40b69f0a5676) service to localhost/127.0.0.1:46645 2024-11-08T11:21:09,727 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T11:21:09,728 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/dfs/data/data1/current/BP-503676101-172.17.0.2-1731064625816 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T11:21:09,729 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/cluster_0273cc36-dce6-592e-7389-005da830abc5/dfs/data/data2/current/BP-503676101-172.17.0.2-1731064625816 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T11:21:09,729 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T11:21:09,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T11:21:09,738 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T11:21:09,738 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T11:21:09,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T11:21:09,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64e48086-d951-ffd0-9a80-674bd6fefd86/hadoop.log.dir/,STOPPED} 2024-11-08T11:21:09,750 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-08T11:21:09,791 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down