2024-11-07 14:17:30,386 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-07 14:17:30,424 main DEBUG Took 0.032205 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 14:17:30,425 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 14:17:30,426 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 14:17:30,456 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 14:17:30,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,499 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 14:17:30,539 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,542 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,543 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,544 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,550 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,551 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,553 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,554 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,555 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,556 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,558 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,558 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,559 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:17:30,560 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,563 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 14:17:30,567 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:17:30,569 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 14:17:30,574 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 14:17:30,575 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 14:17:30,577 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 14:17:30,577 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 14:17:30,597 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 14:17:30,607 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 14:17:30,609 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 14:17:30,611 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 14:17:30,613 main DEBUG createAppenders(={Console}) 2024-11-07 14:17:30,616 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-07 14:17:30,617 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-07 14:17:30,618 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-07 14:17:30,619 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 14:17:30,619 main DEBUG OutputStream closed 2024-11-07 14:17:30,620 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 14:17:30,620 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 14:17:30,620 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-07 14:17:30,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 14:17:30,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 14:17:30,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 14:17:30,797 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 14:17:30,798 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 14:17:30,798 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 14:17:30,798 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 14:17:30,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 14:17:30,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 14:17:30,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 14:17:30,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 14:17:30,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 14:17:30,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 14:17:30,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 14:17:30,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 14:17:30,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 14:17:30,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 14:17:30,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 14:17:30,813 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 14:17:30,814 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-07 14:17:30,814 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 14:17:30,815 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-07T14:17:31,161 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-07 14:17:31,167 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 14:17:31,168 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T14:17:31,321 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a 2024-11-07T14:17:31,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T14:17:31,395 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a, deleteOnExit=true 2024-11-07T14:17:31,396 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-07T14:17:31,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/test.cache.data in system properties and HBase conf 2024-11-07T14:17:31,398 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T14:17:31,400 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/hadoop.log.dir in system properties and HBase conf 2024-11-07T14:17:31,401 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T14:17:31,402 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T14:17:31,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-07T14:17:31,608 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T14:17:31,810 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T14:17:31,817 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T14:17:31,818 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T14:17:31,821 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T14:17:31,823 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T14:17:31,825 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T14:17:31,827 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T14:17:31,828 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T14:17:31,831 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T14:17:31,834 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T14:17:31,835 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/nfs.dump.dir in system properties and HBase conf 2024-11-07T14:17:31,835 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/java.io.tmpdir in system properties and HBase conf 2024-11-07T14:17:31,836 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T14:17:31,837 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T14:17:31,847 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T14:17:33,423 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T14:17:33,597 INFO [Time-limited test {}] log.Log(170): Logging initialized @4150ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T14:17:33,735 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T14:17:33,932 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T14:17:34,008 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T14:17:34,008 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T14:17:34,010 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T14:17:34,043 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T14:17:34,062 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@634f7b0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/hadoop.log.dir/,AVAILABLE} 2024-11-07T14:17:34,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a37b7ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T14:17:34,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27d34e10{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/java.io.tmpdir/jetty-localhost-38427-hadoop-hdfs-3_4_1-tests_jar-_-any-4527592944607294769/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T14:17:34,466 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c62dc53{HTTP/1.1, (http/1.1)}{localhost:38427} 2024-11-07T14:17:34,466 INFO [Time-limited test {}] server.Server(415): Started @5021ms 2024-11-07T14:17:35,293 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T14:17:35,303 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T14:17:35,326 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T14:17:35,326 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T14:17:35,327 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T14:17:35,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@137d3cad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/hadoop.log.dir/,AVAILABLE} 2024-11-07T14:17:35,330 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31b28bed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T14:17:35,526 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6796851a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/java.io.tmpdir/jetty-localhost-46455-hadoop-hdfs-3_4_1-tests_jar-_-any-10163105196544256548/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T14:17:35,527 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@55c2c7f2{HTTP/1.1, (http/1.1)}{localhost:46455} 2024-11-07T14:17:35,528 INFO [Time-limited test {}] server.Server(415): Started @6083ms 2024-11-07T14:17:35,612 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T14:17:36,560 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/dfs/data/data1/current/BP-730893796-172.17.0.2-1730989052927/current, will proceed with Du for space computation calculation, 2024-11-07T14:17:36,568 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/dfs/data/data2/current/BP-730893796-172.17.0.2-1730989052927/current, will proceed with Du for space computation calculation, 2024-11-07T14:17:36,667 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T14:17:36,746 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x781875feff724ca with lease ID 0xebf2abb1d0f06740: Processing first storage report for DS-a78f6041-e21d-4339-aad2-d3053c6739a3 from datanode DatanodeRegistration(127.0.0.1:37655, datanodeUuid=b9e8e1f7-722c-4be9-9307-8e91f8789de8, infoPort=45499, infoSecurePort=0, ipcPort=46443, storageInfo=lv=-57;cid=testClusterID;nsid=154627698;c=1730989052927) 2024-11-07T14:17:36,748 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x781875feff724ca with lease ID 0xebf2abb1d0f06740: from storage DS-a78f6041-e21d-4339-aad2-d3053c6739a3 node DatanodeRegistration(127.0.0.1:37655, datanodeUuid=b9e8e1f7-722c-4be9-9307-8e91f8789de8, infoPort=45499, infoSecurePort=0, ipcPort=46443, storageInfo=lv=-57;cid=testClusterID;nsid=154627698;c=1730989052927), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T14:17:36,748 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x781875feff724ca with lease ID 0xebf2abb1d0f06740: Processing first storage report for DS-2d0e9224-18c4-443f-bda6-adec2ce79bae from datanode DatanodeRegistration(127.0.0.1:37655, datanodeUuid=b9e8e1f7-722c-4be9-9307-8e91f8789de8, infoPort=45499, infoSecurePort=0, ipcPort=46443, storageInfo=lv=-57;cid=testClusterID;nsid=154627698;c=1730989052927) 2024-11-07T14:17:36,749 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x781875feff724ca with lease ID 0xebf2abb1d0f06740: from storage DS-2d0e9224-18c4-443f-bda6-adec2ce79bae node DatanodeRegistration(127.0.0.1:37655, datanodeUuid=b9e8e1f7-722c-4be9-9307-8e91f8789de8, infoPort=45499, infoSecurePort=0, ipcPort=46443, storageInfo=lv=-57;cid=testClusterID;nsid=154627698;c=1730989052927), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T14:17:36,831 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a 2024-11-07T14:17:37,067 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/zookeeper_0, clientPort=49795, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T14:17:37,084 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49795 2024-11-07T14:17:37,109 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:37,115 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741825_1001 (size=7) 2024-11-07T14:17:37,895 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 with version=8 2024-11-07T14:17:37,895 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/hbase-staging 2024-11-07T14:17:38,085 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T14:17:38,506 INFO [Time-limited test {}] client.ConnectionUtils(129): master/69430dbfd73f:0 server-side Connection retries=9 2024-11-07T14:17:38,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:17:38,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T14:17:38,535 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T14:17:38,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:17:38,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T14:17:38,832 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T14:17:38,951 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T14:17:38,966 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T14:17:38,973 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T14:17:39,014 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12638 (auto-detected) 2024-11-07T14:17:39,016 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T14:17:39,062 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42745 2024-11-07T14:17:39,075 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:39,079 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:39,099 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:42745 connecting to ZooKeeper ensemble=127.0.0.1:49795 2024-11-07T14:17:39,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427450x0, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T14:17:39,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42745-0x1018318593e0000 connected 2024-11-07T14:17:39,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T14:17:39,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:17:39,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T14:17:39,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42745 2024-11-07T14:17:39,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42745 2024-11-07T14:17:39,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42745 2024-11-07T14:17:39,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42745 2024-11-07T14:17:39,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42745 2024-11-07T14:17:39,500 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452, hbase.cluster.distributed=false 2024-11-07T14:17:39,682 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/69430dbfd73f:0 server-side Connection retries=9 2024-11-07T14:17:39,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:17:39,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T14:17:39,683 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T14:17:39,684 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:17:39,684 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T14:17:39,687 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T14:17:39,691 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T14:17:39,692 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36593 2024-11-07T14:17:39,695 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T14:17:39,719 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T14:17:39,721 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:39,725 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:39,731 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36593 connecting to ZooKeeper ensemble=127.0.0.1:49795 2024-11-07T14:17:39,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365930x0, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T14:17:39,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365930x0, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T14:17:39,736 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36593-0x1018318593e0001 connected 2024-11-07T14:17:39,742 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:17:39,744 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T14:17:39,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36593 2024-11-07T14:17:39,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36593 2024-11-07T14:17:39,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36593 2024-11-07T14:17:39,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36593 2024-11-07T14:17:39,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36593 2024-11-07T14:17:39,830 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/69430dbfd73f,42745,1730989058075 2024-11-07T14:17:39,842 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;69430dbfd73f:42745 2024-11-07T14:17:39,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:17:39,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:17:39,852 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/69430dbfd73f,42745,1730989058075 2024-11-07T14:17:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T14:17:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T14:17:39,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:39,925 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T14:17:39,929 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T14:17:39,932 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/69430dbfd73f,42745,1730989058075 from backup master directory 2024-11-07T14:17:39,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:17:39,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/69430dbfd73f,42745,1730989058075 2024-11-07T14:17:39,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:17:39,940 WARN [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T14:17:39,940 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=69430dbfd73f,42745,1730989058075 2024-11-07T14:17:39,943 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T14:17:39,960 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T14:17:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741826_1002 (size=42) 2024-11-07T14:17:40,081 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/hbase.id with ID: e0cbab97-6468-4c84-a767-f40ccf5f36df 2024-11-07T14:17:40,168 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:17:40,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:40,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:40,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741827_1003 (size=196) 2024-11-07T14:17:40,321 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:17:40,325 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T14:17:40,356 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:17:40,363 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T14:17:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741828_1004 (size=1189) 2024-11-07T14:17:40,488 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store 2024-11-07T14:17:40,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741829_1005 (size=34) 2024-11-07T14:17:40,542 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T14:17:40,543 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:17:40,544 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T14:17:40,544 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:17:40,545 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:17:40,545 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T14:17:40,545 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:17:40,545 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:17:40,546 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T14:17:40,550 WARN [master/69430dbfd73f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/.initializing 2024-11-07T14:17:40,550 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/WALs/69430dbfd73f,42745,1730989058075 2024-11-07T14:17:40,561 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T14:17:40,584 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=69430dbfd73f%2C42745%2C1730989058075, suffix=, logDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/WALs/69430dbfd73f,42745,1730989058075, archiveDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/oldWALs, maxLogs=10 2024-11-07T14:17:40,625 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/WALs/69430dbfd73f,42745,1730989058075/69430dbfd73f%2C42745%2C1730989058075.1730989060595, exclude list is [], retry=0 2024-11-07T14:17:40,675 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37655,DS-a78f6041-e21d-4339-aad2-d3053c6739a3,DISK] 2024-11-07T14:17:40,680 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T14:17:40,740 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/WALs/69430dbfd73f,42745,1730989058075/69430dbfd73f%2C42745%2C1730989058075.1730989060595 2024-11-07T14:17:40,741 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45499:45499)] 2024-11-07T14:17:40,742 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:17:40,742 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:17:40,747 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,748 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T14:17:40,849 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:40,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:17:40,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,864 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T14:17:40,865 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:40,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:17:40,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T14:17:40,872 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:40,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:17:40,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T14:17:40,877 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:40,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:17:40,883 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,884 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,900 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T14:17:40,909 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:17:40,926 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:17:40,928 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66562589, jitterRate=-0.008140131831169128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T14:17:40,935 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T14:17:40,938 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T14:17:40,991 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d43c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:17:41,039 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-07T14:17:41,055 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T14:17:41,055 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T14:17:41,058 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T14:17:41,059 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T14:17:41,069 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 9 msec 2024-11-07T14:17:41,069 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T14:17:41,113 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T14:17:41,129 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T14:17:41,133 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-07T14:17:41,137 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T14:17:41,138 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T14:17:41,140 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-07T14:17:41,142 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T14:17:41,146 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T14:17:41,148 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-07T14:17:41,149 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T14:17:41,156 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T14:17:41,170 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T14:17:41,173 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T14:17:41,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T14:17:41,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T14:17:41,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:41,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:41,180 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=69430dbfd73f,42745,1730989058075, sessionid=0x1018318593e0000, setting cluster-up flag (Was=false) 2024-11-07T14:17:41,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:41,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:41,203 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T14:17:41,205 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=69430dbfd73f,42745,1730989058075 2024-11-07T14:17:41,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:41,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:17:41,223 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T14:17:41,225 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=69430dbfd73f,42745,1730989058075 2024-11-07T14:17:41,340 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-07T14:17:41,348 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-07T14:17:41,348 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;69430dbfd73f:36593 2024-11-07T14:17:41,351 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1008): ClusterId : e0cbab97-6468-4c84-a767-f40ccf5f36df 2024-11-07T14:17:41,352 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T14:17:41,354 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T14:17:41,360 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T14:17:41,360 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T14:17:41,360 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 69430dbfd73f,42745,1730989058075 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T14:17:41,364 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/69430dbfd73f:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:17:41,364 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/69430dbfd73f:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:17:41,364 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T14:17:41,365 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/69430dbfd73f:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:17:41,365 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/69430dbfd73f:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:17:41,365 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/69430dbfd73f:0, corePoolSize=10, maxPoolSize=10 2024-11-07T14:17:41,365 DEBUG [RS:0;69430dbfd73f:36593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6632d213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:17:41,365 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,366 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/69430dbfd73f:0, corePoolSize=2, maxPoolSize=2 2024-11-07T14:17:41,366 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,371 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730989091371 2024-11-07T14:17:41,373 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T14:17:41,374 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-07T14:17:41,375 DEBUG [RS:0;69430dbfd73f:36593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edf4f87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=69430dbfd73f/172.17.0.2:0 2024-11-07T14:17:41,379 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-07T14:17:41,373 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T14:17:41,380 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:41,381 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T14:17:41,382 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T14:17:41,379 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-07T14:17:41,386 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-07T14:17:41,386 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T14:17:41,387 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T14:17:41,387 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T14:17:41,388 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T14:17:41,391 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3073): reportForDuty to master=69430dbfd73f,42745,1730989058075 with isa=69430dbfd73f/172.17.0.2:36593, startcode=1730989059679 2024-11-07T14:17:41,392 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,396 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T14:17:41,398 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T14:17:41,398 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T14:17:41,407 DEBUG [RS:0;69430dbfd73f:36593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T14:17:41,409 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T14:17:41,411 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T14:17:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741831_1007 (size=1039) 2024-11-07T14:17:41,418 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-07T14:17:41,421 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/69430dbfd73f:0:becomeActiveMaster-HFileCleaner.large.0-1730989061414,5,FailOnTimeoutGroup] 2024-11-07T14:17:41,421 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:17:41,423 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/69430dbfd73f:0:becomeActiveMaster-HFileCleaner.small.0-1730989061421,5,FailOnTimeoutGroup] 2024-11-07T14:17:41,432 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,433 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T14:17:41,436 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,436 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,475 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46553, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T14:17:41,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741832_1008 (size=32) 2024-11-07T14:17:41,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:17:41,485 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42745 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 69430dbfd73f,36593,1730989059679 2024-11-07T14:17:41,490 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42745 {}] master.ServerManager(486): Registering regionserver=69430dbfd73f,36593,1730989059679 2024-11-07T14:17:41,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T14:17:41,512 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:17:41,512 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:41411 2024-11-07T14:17:41,512 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-07T14:17:41,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T14:17:41,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:41,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:17:41,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T14:17:41,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T14:17:41,519 DEBUG [RS:0;69430dbfd73f:36593 {}] zookeeper.ZKUtil(111): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/69430dbfd73f,36593,1730989059679 2024-11-07T14:17:41,519 WARN [RS:0;69430dbfd73f:36593 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T14:17:41,519 INFO [RS:0;69430dbfd73f:36593 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T14:17:41,520 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679 2024-11-07T14:17:41,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T14:17:41,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:41,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:17:41,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T14:17:41,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T14:17:41,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:17:41,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:17:41,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [69430dbfd73f,36593,1730989059679] 2024-11-07T14:17:41,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740 2024-11-07T14:17:41,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740 2024-11-07T14:17:41,536 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T14:17:41,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T14:17:41,555 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:17:41,557 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74333333, jitterRate=0.10765297710895538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T14:17:41,561 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-07T14:17:41,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T14:17:41,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T14:17:41,565 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T14:17:41,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T14:17:41,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T14:17:41,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T14:17:41,573 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T14:17:41,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T14:17:41,577 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T14:17:41,577 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-07T14:17:41,584 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T14:17:41,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T14:17:41,598 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T14:17:41,601 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T14:17:41,605 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T14:17:41,614 INFO [RS:0;69430dbfd73f:36593 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T14:17:41,614 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,616 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-07T14:17:41,626 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,626 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,626 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,627 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,627 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,627 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,627 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/69430dbfd73f:0, corePoolSize=2, maxPoolSize=2 2024-11-07T14:17:41,627 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,628 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,628 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,628 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,628 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/69430dbfd73f:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:17:41,629 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/69430dbfd73f:0, corePoolSize=3, maxPoolSize=3 2024-11-07T14:17:41,631 DEBUG [RS:0;69430dbfd73f:36593 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/69430dbfd73f:0, corePoolSize=3, maxPoolSize=3 2024-11-07T14:17:41,636 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,636 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,637 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,637 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,637 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,36593,1730989059679-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T14:17:41,673 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T14:17:41,676 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,36593,1730989059679-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:17:41,704 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.Replication(204): 69430dbfd73f,36593,1730989059679 started 2024-11-07T14:17:41,704 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1767): Serving as 69430dbfd73f,36593,1730989059679, RpcServer on 69430dbfd73f/172.17.0.2:36593, sessionid=0x1018318593e0001 2024-11-07T14:17:41,705 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T14:17:41,705 DEBUG [RS:0;69430dbfd73f:36593 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 69430dbfd73f,36593,1730989059679 2024-11-07T14:17:41,705 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '69430dbfd73f,36593,1730989059679' 2024-11-07T14:17:41,705 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T14:17:41,706 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T14:17:41,707 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T14:17:41,707 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T14:17:41,707 DEBUG [RS:0;69430dbfd73f:36593 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 69430dbfd73f,36593,1730989059679 2024-11-07T14:17:41,708 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '69430dbfd73f,36593,1730989059679' 2024-11-07T14:17:41,708 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T14:17:41,708 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T14:17:41,709 DEBUG [RS:0;69430dbfd73f:36593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T14:17:41,709 INFO [RS:0;69430dbfd73f:36593 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T14:17:41,709 INFO [RS:0;69430dbfd73f:36593 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T14:17:41,752 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:42,754 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:43,754 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:44,755 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:45,756 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:46,757 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:47,758 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:48,759 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:48,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T14:17:48,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T14:17:49,759 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:50,760 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:51,761 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:52,762 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:53,762 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:54,763 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:55,764 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:56,764 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:57,765 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:58,766 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:17:59,767 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:00,768 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:01,768 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:02,769 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:03,770 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:04,771 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:05,772 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:06,772 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:06,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:18:07,248 ERROR [LeaseRenewer:jenkins@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:41411,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:18:07,773 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:08,774 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:09,775 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:10,775 WARN [69430dbfd73f:42745 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T14:18:11,717 INFO [RS:0;69430dbfd73f:36593 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T14:18:11,723 INFO [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=69430dbfd73f%2C36593%2C1730989059679, suffix=, logDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679, archiveDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/oldWALs, maxLogs=32 2024-11-07T14:18:11,742 DEBUG [RS:0;69430dbfd73f:36593 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679/69430dbfd73f%2C36593%2C1730989059679.1730989091725, exclude list is [], retry=0 2024-11-07T14:18:11,749 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37655,DS-a78f6041-e21d-4339-aad2-d3053c6739a3,DISK] 2024-11-07T14:18:11,753 INFO [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679/69430dbfd73f%2C36593%2C1730989059679.1730989091725 2024-11-07T14:18:11,754 DEBUG [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45499:45499)] 2024-11-07T14:18:11,777 DEBUG [69430dbfd73f:42745 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T14:18:11,782 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:11,788 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 69430dbfd73f,36593,1730989059679, state=OPENING 2024-11-07T14:18:11,797 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T14:18:11,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:18:11,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:18:11,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:18:11,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:18:11,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:11,985 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:11,988 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T14:18:11,992 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51992, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T14:18:12,005 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-07T14:18:12,005 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T14:18:12,005 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T14:18:12,010 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=69430dbfd73f%2C36593%2C1730989059679.meta, suffix=.meta, logDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679, archiveDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/oldWALs, maxLogs=32 2024-11-07T14:18:12,029 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679/69430dbfd73f%2C36593%2C1730989059679.meta.1730989092012.meta, exclude list is [], retry=0 2024-11-07T14:18:12,034 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37655,DS-a78f6041-e21d-4339-aad2-d3053c6739a3,DISK] 2024-11-07T14:18:12,039 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679/69430dbfd73f%2C36593%2C1730989059679.meta.1730989092012.meta 2024-11-07T14:18:12,039 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45499:45499)] 2024-11-07T14:18:12,040 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:12,042 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T14:18:12,111 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T14:18:12,118 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T14:18:12,123 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T14:18:12,124 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:12,124 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-07T14:18:12,124 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-07T14:18:12,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T14:18:12,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T14:18:12,132 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:12,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:18:12,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T14:18:12,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T14:18:12,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:12,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:18:12,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T14:18:12,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T14:18:12,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:12,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:18:12,142 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740 2024-11-07T14:18:12,146 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740 2024-11-07T14:18:12,149 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T14:18:12,153 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T14:18:12,155 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70442845, jitterRate=0.0496801882982254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T14:18:12,158 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T14:18:12,166 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730989091978 2024-11-07T14:18:12,180 DEBUG [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T14:18:12,180 INFO [RS_OPEN_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-07T14:18:12,182 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:12,184 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 69430dbfd73f,36593,1730989059679, state=OPEN 2024-11-07T14:18:12,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T14:18:12,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T14:18:12,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:18:12,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:18:12,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T14:18:12,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=69430dbfd73f,36593,1730989059679 in 389 msec 2024-11-07T14:18:12,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T14:18:12,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.6120 sec 2024-11-07T14:18:12,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 30.9350 sec 2024-11-07T14:18:12,211 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730989092211, completionTime=-1 2024-11-07T14:18:12,211 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T14:18:12,212 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-07T14:18:12,259 DEBUG [hconnection-0x556a1b08-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:12,262 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51998, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:12,276 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-07T14:18:12,277 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730989152276 2024-11-07T14:18:12,277 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730989212277 2024-11-07T14:18:12,277 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 65 msec 2024-11-07T14:18:12,303 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,42745,1730989058075-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:18:12,303 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,42745,1730989058075-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:18:12,303 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,42745,1730989058075-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:18:12,305 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-69430dbfd73f:42745, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:18:12,305 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T14:18:12,312 DEBUG [master/69430dbfd73f:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-07T14:18:12,314 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-07T14:18:12,316 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T14:18:12,323 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-07T14:18:12,327 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:12,328 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:12,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:12,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741835_1011 (size=358) 2024-11-07T14:18:12,347 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f292f0dddc0193d61d9f927f4dc3af4c, NAME => 'hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:12,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741836_1012 (size=42) 2024-11-07T14:18:12,366 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:12,366 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing f292f0dddc0193d61d9f927f4dc3af4c, disabling compactions & flushes 2024-11-07T14:18:12,366 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,366 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,366 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. after waiting 0 ms 2024-11-07T14:18:12,366 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,366 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,366 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for f292f0dddc0193d61d9f927f4dc3af4c: 2024-11-07T14:18:12,369 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:12,378 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1730989092370"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989092370"}]},"ts":"1730989092370"} 2024-11-07T14:18:12,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:12,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:12,414 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989092411"}]},"ts":"1730989092411"} 2024-11-07T14:18:12,420 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-07T14:18:12,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=f292f0dddc0193d61d9f927f4dc3af4c, ASSIGN}] 2024-11-07T14:18:12,429 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=f292f0dddc0193d61d9f927f4dc3af4c, ASSIGN 2024-11-07T14:18:12,431 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=f292f0dddc0193d61d9f927f4dc3af4c, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:12,582 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=f292f0dddc0193d61d9f927f4dc3af4c, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:12,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure f292f0dddc0193d61d9f927f4dc3af4c, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:12,745 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:12,753 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => f292f0dddc0193d61d9f927f4dc3af4c, NAME => 'hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:12,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:12,755 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,755 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,758 INFO [StoreOpener-f292f0dddc0193d61d9f927f4dc3af4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,761 INFO [StoreOpener-f292f0dddc0193d61d9f927f4dc3af4c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f292f0dddc0193d61d9f927f4dc3af4c columnFamilyName info 2024-11-07T14:18:12,761 DEBUG [StoreOpener-f292f0dddc0193d61d9f927f4dc3af4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:12,762 INFO [StoreOpener-f292f0dddc0193d61d9f927f4dc3af4c-1 {}] regionserver.HStore(327): Store=f292f0dddc0193d61d9f927f4dc3af4c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:12,764 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,770 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:18:12,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:12,776 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened f292f0dddc0193d61d9f927f4dc3af4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61548903, jitterRate=-0.08284987509250641}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:12,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for f292f0dddc0193d61d9f927f4dc3af4c: 2024-11-07T14:18:12,780 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c., pid=6, masterSystemTime=1730989092745 2024-11-07T14:18:12,785 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,785 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:18:12,786 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=f292f0dddc0193d61d9f927f4dc3af4c, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:12,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T14:18:12,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure f292f0dddc0193d61d9f927f4dc3af4c, server=69430dbfd73f,36593,1730989059679 in 202 msec 2024-11-07T14:18:12,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T14:18:12,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=f292f0dddc0193d61d9f927f4dc3af4c, ASSIGN in 369 msec 2024-11-07T14:18:12,802 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:12,802 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989092802"}]},"ts":"1730989092802"} 2024-11-07T14:18:12,805 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-07T14:18:12,810 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:12,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 494 msec 2024-11-07T14:18:12,827 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-07T14:18:12,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:18:12,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-07T14:18:12,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:18:12,865 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-07T14:18:12,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T14:18:12,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 29 msec 2024-11-07T14:18:12,900 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-07T14:18:12,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T14:18:12,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 19 msec 2024-11-07T14:18:12,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-07T14:18:12,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-07T14:18:12,942 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 33.002sec 2024-11-07T14:18:12,943 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T14:18:12,945 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T14:18:12,946 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T14:18:12,947 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T14:18:12,947 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T14:18:12,948 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,42745,1730989058075-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T14:18:12,948 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,42745,1730989058075-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T14:18:12,956 DEBUG [master/69430dbfd73f:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-07T14:18:12,957 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T14:18:12,958 INFO [master/69430dbfd73f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=69430dbfd73f,42745,1730989058075-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:18:12,978 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cd335b7 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4876bbc7 2024-11-07T14:18:12,979 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-07T14:18:12,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29821dfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:12,993 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T14:18:12,993 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T14:18:13,004 DEBUG [hconnection-0x6265bb37-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:13,015 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52014, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:13,025 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=69430dbfd73f,42745,1730989058075 2024-11-07T14:18:13,047 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=6093 2024-11-07T14:18:13,053 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03603e11 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@62fd2e57 2024-11-07T14:18:13,061 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x03603e11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8b5515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:13,067 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0ddff099 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6dbf8e3 2024-11-07T14:18:13,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a6fdbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:13,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:18:13,089 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54000, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:18:13,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T14:18:13,105 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:13,106 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:13,108 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:13,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-07T14:18:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T14:18:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741837_1013 (size=437) 2024-11-07T14:18:13,125 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => fcb6eac7f983513750598f0dd2c593b3, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:13,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741838_1014 (size=88) 2024-11-07T14:18:13,168 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:13,168 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing fcb6eac7f983513750598f0dd2c593b3, disabling compactions & flushes 2024-11-07T14:18:13,168 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,168 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,168 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. after waiting 0 ms 2024-11-07T14:18:13,168 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,168 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,168 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for fcb6eac7f983513750598f0dd2c593b3: 2024-11-07T14:18:13,172 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:13,172 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1730989093172"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989093172"}]},"ts":"1730989093172"} 2024-11-07T14:18:13,178 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:13,182 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:13,182 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989093182"}]},"ts":"1730989093182"} 2024-11-07T14:18:13,187 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:13,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fcb6eac7f983513750598f0dd2c593b3, ASSIGN}] 2024-11-07T14:18:13,196 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fcb6eac7f983513750598f0dd2c593b3, ASSIGN 2024-11-07T14:18:13,197 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fcb6eac7f983513750598f0dd2c593b3, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:13,348 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fcb6eac7f983513750598f0dd2c593b3, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:13,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure fcb6eac7f983513750598f0dd2c593b3, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:13,510 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:13,518 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,518 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => fcb6eac7f983513750598f0dd2c593b3, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:13,519 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,519 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:13,519 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,519 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,522 INFO [StoreOpener-fcb6eac7f983513750598f0dd2c593b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,525 INFO [StoreOpener-fcb6eac7f983513750598f0dd2c593b3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcb6eac7f983513750598f0dd2c593b3 columnFamilyName testFamily 2024-11-07T14:18:13,525 DEBUG [StoreOpener-fcb6eac7f983513750598f0dd2c593b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:13,527 INFO [StoreOpener-fcb6eac7f983513750598f0dd2c593b3-1 {}] regionserver.HStore(327): Store=fcb6eac7f983513750598f0dd2c593b3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:13,528 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,529 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,533 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:18:13,545 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:13,547 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened fcb6eac7f983513750598f0dd2c593b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60921151, jitterRate=-0.09220410883426666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:13,549 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for fcb6eac7f983513750598f0dd2c593b3: 2024-11-07T14:18:13,551 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3., pid=11, masterSystemTime=1730989093510 2024-11-07T14:18:13,555 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,555 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:18:13,560 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fcb6eac7f983513750598f0dd2c593b3, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:13,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-07T14:18:13,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure fcb6eac7f983513750598f0dd2c593b3, server=69430dbfd73f,36593,1730989059679 in 209 msec 2024-11-07T14:18:13,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T14:18:13,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fcb6eac7f983513750598f0dd2c593b3, ASSIGN in 377 msec 2024-11-07T14:18:13,577 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:13,578 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989093577"}]},"ts":"1730989093577"} 2024-11-07T14:18:13,581 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:13,586 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:13,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 485 msec 2024-11-07T14:18:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T14:18:14,127 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-07T14:18:14,128 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:14,129 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:14,134 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:14,135 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:14,135 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-07T14:18:14,138 DEBUG [hconnection-0x26bac998-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:14,140 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52016, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:14,157 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:14,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T14:18:14,181 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:18:14,183 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x03603e11 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:18:14,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:14,191 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52022, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:18:14,204 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T14:18:14,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668939 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:14,207 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T14:18:14,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668939 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:14,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628668939 2024-11-07T14:18:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52022 deadline: 1730989097206, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:14,233 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668939 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:14,236 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T14:18:14,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668940 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:14,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T14:18:14,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:14,240 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:14,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668941 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:14,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628668941 2024-11-07T14:18:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52022 deadline: 1730989097239, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:14,242 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668941 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:14,244 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:14,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668942 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:14,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4907802727628668942 2024-11-07T14:18:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52022 deadline: 1730989097243, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:14,247 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:14,246 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668942 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:14,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668943 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:14,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4907802727628668943 2024-11-07T14:18:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52022 deadline: 1730989097247, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:14,251 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668943 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:14,252 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:14,253 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0ddff099 to 127.0.0.1:49795 2024-11-07T14:18:14,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:14,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:14,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:14,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:14,282 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x03603e11 to 127.0.0.1:49795 2024-11-07T14:18:14,303 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=232 (was 219) Potentially hanging thread: RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49795@0x0ddff099 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$732/0x00007fe214b43be8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49795@0x0ddff099-SendThread(127.0.0.1:49795) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49795@0x03603e11-SendThread(127.0.0.1:49795) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49795@0x0ddff099-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49795@0x03603e11-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1478915338_22 at /127.0.0.1:58484 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.newPromise(AbstractChannelHandlerContext.java:1011) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:560) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.close(DefaultChannelPipeline.java:906) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel.close(AbstractChannel.java:243) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown0(NettyRpcConnection.java:139) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection$$Lambda$803/0x00007fe214b77540.run(Unknown Source) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49795@0x03603e11 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$732/0x00007fe214b43be8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x6265bb37-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=454 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=6040 (was 6093) 2024-11-07T14:18:14,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=232, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=6039 2024-11-07T14:18:14,324 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2bf5a78e to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@123ce537 2024-11-07T14:18:14,329 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x2bf5a78e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8a770d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:14,331 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c3f467b to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a85729a 2024-11-07T14:18:14,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4a7b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:14,347 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:14,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T14:18:14,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:14,352 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:14,352 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-07T14:18:14,354 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T14:18:14,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741839_1015 (size=423) 2024-11-07T14:18:14,780 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 89b3721db112bdfe9245cf965df47224, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741840_1016 (size=74) 2024-11-07T14:18:15,197 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:15,197 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 89b3721db112bdfe9245cf965df47224, disabling compactions & flushes 2024-11-07T14:18:15,197 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,197 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,197 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. after waiting 0 ms 2024-11-07T14:18:15,197 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,197 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,197 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 89b3721db112bdfe9245cf965df47224: 2024-11-07T14:18:15,200 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:15,200 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1730989095200"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989095200"}]},"ts":"1730989095200"} 2024-11-07T14:18:15,204 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:15,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:15,209 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989095209"}]},"ts":"1730989095209"} 2024-11-07T14:18:15,212 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:15,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=89b3721db112bdfe9245cf965df47224, ASSIGN}] 2024-11-07T14:18:15,220 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=89b3721db112bdfe9245cf965df47224, ASSIGN 2024-11-07T14:18:15,222 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=89b3721db112bdfe9245cf965df47224, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T14:18:15,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=89b3721db112bdfe9245cf965df47224, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:15,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 89b3721db112bdfe9245cf965df47224, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:15,530 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:15,536 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,536 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => 89b3721db112bdfe9245cf965df47224, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:15,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:15,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for 89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for 89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,540 INFO [StoreOpener-89b3721db112bdfe9245cf965df47224-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,542 INFO [StoreOpener-89b3721db112bdfe9245cf965df47224-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89b3721db112bdfe9245cf965df47224 columnFamilyName testFamily 2024-11-07T14:18:15,542 DEBUG [StoreOpener-89b3721db112bdfe9245cf965df47224-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:15,543 INFO [StoreOpener-89b3721db112bdfe9245cf965df47224-1 {}] regionserver.HStore(327): Store=89b3721db112bdfe9245cf965df47224/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:15,545 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for 89b3721db112bdfe9245cf965df47224 2024-11-07T14:18:15,553 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:15,554 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened 89b3721db112bdfe9245cf965df47224; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58827570, jitterRate=-0.12340089678764343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:15,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for 89b3721db112bdfe9245cf965df47224: 2024-11-07T14:18:15,558 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224., pid=14, masterSystemTime=1730989095530 2024-11-07T14:18:15,561 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,561 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:18:15,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=89b3721db112bdfe9245cf965df47224, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:15,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T14:18:15,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 89b3721db112bdfe9245cf965df47224, server=69430dbfd73f,36593,1730989059679 in 190 msec 2024-11-07T14:18:15,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-07T14:18:15,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=89b3721db112bdfe9245cf965df47224, ASSIGN in 353 msec 2024-11-07T14:18:15,575 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:15,575 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989095575"}]},"ts":"1730989095575"} 2024-11-07T14:18:15,578 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:15,582 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:15,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.2350 sec 2024-11-07T14:18:16,369 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T14:18:16,372 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54008, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T14:18:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T14:18:17,365 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-07T14:18:17,365 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:17,365 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:17,370 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:17,370 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:17,370 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T14:18:17,373 DEBUG [hconnection-0x71197bc8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:17,375 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52028, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:17,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:17,394 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:18:17,396 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:18:17,662 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:18:17,694 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-07T14:18:17,695 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-07T14:18:17,696 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T14:18:17,697 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-07T14:18:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T14:18:18,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:18,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T14:18:18,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:18,948 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-07T14:18:18,949 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:18,950 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T14:18:18,950 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T14:18:18,950 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T14:18:18,950 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-07T14:18:24,428 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:18:26,414 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11024: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11024: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:18:26,415 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T14:18:26.415Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11024: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11024: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1978ms, rpcTimeout=1969ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:18:26,417 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9022 2024-11-07T14:18:26,418 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c3f467b to 127.0.0.1:49795 2024-11-07T14:18:26,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:26,418 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:26,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:26,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:26,419 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2bf5a78e to 127.0.0.1:49795 2024-11-07T14:18:26,440 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=223 (was 232), OpenFileDescriptor=449 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=5946 (was 6039) 2024-11-07T14:18:26,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=5945 2024-11-07T14:18:26,455 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74fc6a68 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40b4a257 2024-11-07T14:18:26,460 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x74fc6a68 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1984d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:26,462 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d0b571f to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ee9ff90 2024-11-07T14:18:26,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29516d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:26,469 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:26,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T14:18:26,473 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:26,473 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:26,473 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-07T14:18:26,475 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T14:18:26,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741841_1017 (size=421) 2024-11-07T14:18:26,934 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f18b2a090aac227c596c7ae1e30557ce, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:26,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741842_1018 (size=72) 2024-11-07T14:18:26,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:26,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing f18b2a090aac227c596c7ae1e30557ce, disabling compactions & flushes 2024-11-07T14:18:26,952 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:26,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:26,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. after waiting 0 ms 2024-11-07T14:18:26,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:26,952 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:26,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for f18b2a090aac227c596c7ae1e30557ce: 2024-11-07T14:18:26,954 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:26,955 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730989106954"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989106954"}]},"ts":"1730989106954"} 2024-11-07T14:18:26,958 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:26,960 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:26,961 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989106960"}]},"ts":"1730989106960"} 2024-11-07T14:18:26,966 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:26,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f18b2a090aac227c596c7ae1e30557ce, ASSIGN}] 2024-11-07T14:18:26,974 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f18b2a090aac227c596c7ae1e30557ce, ASSIGN 2024-11-07T14:18:26,977 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f18b2a090aac227c596c7ae1e30557ce, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:27,128 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=f18b2a090aac227c596c7ae1e30557ce, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:27,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure f18b2a090aac227c596c7ae1e30557ce, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:27,287 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:27,306 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:27,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => f18b2a090aac227c596c7ae1e30557ce, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:27,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:27,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,312 INFO [StoreOpener-f18b2a090aac227c596c7ae1e30557ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,319 INFO [StoreOpener-f18b2a090aac227c596c7ae1e30557ce-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f18b2a090aac227c596c7ae1e30557ce columnFamilyName testFamily 2024-11-07T14:18:27,320 DEBUG [StoreOpener-f18b2a090aac227c596c7ae1e30557ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:27,322 INFO [StoreOpener-f18b2a090aac227c596c7ae1e30557ce-1 {}] regionserver.HStore(327): Store=f18b2a090aac227c596c7ae1e30557ce/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:27,324 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,325 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:18:27,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:27,341 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened f18b2a090aac227c596c7ae1e30557ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74721725, jitterRate=0.11344046890735626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:27,342 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for f18b2a090aac227c596c7ae1e30557ce: 2024-11-07T14:18:27,344 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce., pid=17, masterSystemTime=1730989107286 2024-11-07T14:18:27,348 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:27,348 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:18:27,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=f18b2a090aac227c596c7ae1e30557ce, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:27,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-07T14:18:27,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure f18b2a090aac227c596c7ae1e30557ce, server=69430dbfd73f,36593,1730989059679 in 221 msec 2024-11-07T14:18:27,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T14:18:27,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:27,365 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989107365"}]},"ts":"1730989107365"} 2024-11-07T14:18:27,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f18b2a090aac227c596c7ae1e30557ce, ASSIGN in 386 msec 2024-11-07T14:18:27,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:27,373 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:27,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 904 msec 2024-11-07T14:18:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T14:18:27,480 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-07T14:18:27,480 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:27,481 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:27,486 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:27,487 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:27,487 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T14:18:27,490 DEBUG [hconnection-0x38f98993-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:27,493 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50788, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:27,514 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:27,519 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:18:27,519 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:18:27,521 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x74fc6a68 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:18:27,522 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x74fc6a68 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:27,536 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50796, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:18:27,543 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:18:27,544 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668955 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:27,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:18:27,551 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668955 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:27,552 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T14:18:28,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T14:18:28,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:29,699 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-07T14:18:30,928 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:18:32,845 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:18:36,555 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9004 2024-11-07T14:18:36,555 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668955 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:36,555 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d0b571f to 127.0.0.1:49795 2024-11-07T14:18:36,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:36,556 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:36,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628668955' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T14:18:36,556 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:36,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:36,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:50796 deadline: 1730989122555, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628668955' does not exist 2024-11-07T14:18:36,556 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74fc6a68 to 127.0.0.1:49795 2024-11-07T14:18:36,560 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 69430dbfd73f,36593,1730989059679 for closing scanner id = 4907802727628668955 for f18b2a090aac227c596c7ae1e30557ce of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628668955' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:18:36,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=225 (was 223) - Thread LEAK? -, OpenFileDescriptor=452 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 446), ProcessCount=11 (was 11), AvailableMemoryMB=5918 (was 5945) 2024-11-07T14:18:36,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=225, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=5918 2024-11-07T14:18:36,583 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54c4b0b3 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@472467d6 2024-11-07T14:18:36,588 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x54c4b0b3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4428f52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:36,589 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ee57684 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@476c684c 2024-11-07T14:18:36,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36dc193a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:36,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T14:18:36,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:36,602 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:36,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-07T14:18:36,604 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T14:18:36,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741843_1019 (size=432) 2024-11-07T14:18:36,824 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:18:37,026 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0e797cba1bb85ec7f7b6bb605c8e5361, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:37,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741844_1020 (size=83) 2024-11-07T14:18:37,036 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:37,036 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 0e797cba1bb85ec7f7b6bb605c8e5361, disabling compactions & flushes 2024-11-07T14:18:37,036 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,036 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,036 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. after waiting 0 ms 2024-11-07T14:18:37,036 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,036 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,036 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0e797cba1bb85ec7f7b6bb605c8e5361: 2024-11-07T14:18:37,038 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:37,039 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1730989117038"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989117038"}]},"ts":"1730989117038"} 2024-11-07T14:18:37,046 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:37,047 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:37,047 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989117047"}]},"ts":"1730989117047"} 2024-11-07T14:18:37,050 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:37,053 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:18:37,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=0e797cba1bb85ec7f7b6bb605c8e5361, ASSIGN}] 2024-11-07T14:18:37,059 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=0e797cba1bb85ec7f7b6bb605c8e5361, ASSIGN 2024-11-07T14:18:37,061 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=0e797cba1bb85ec7f7b6bb605c8e5361, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:37,212 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=0e797cba1bb85ec7f7b6bb605c8e5361, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:37,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 0e797cba1bb85ec7f7b6bb605c8e5361, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:37,369 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:37,377 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,378 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => 0e797cba1bb85ec7f7b6bb605c8e5361, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:37,378 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,378 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:37,378 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for 0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,378 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for 0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,385 INFO [StoreOpener-0e797cba1bb85ec7f7b6bb605c8e5361-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,388 INFO [StoreOpener-0e797cba1bb85ec7f7b6bb605c8e5361-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e797cba1bb85ec7f7b6bb605c8e5361 columnFamilyName testFamily 2024-11-07T14:18:37,388 DEBUG [StoreOpener-0e797cba1bb85ec7f7b6bb605c8e5361-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:37,389 INFO [StoreOpener-0e797cba1bb85ec7f7b6bb605c8e5361-1 {}] regionserver.HStore(327): Store=0e797cba1bb85ec7f7b6bb605c8e5361/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:37,391 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,392 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,396 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for 0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:18:37,402 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:37,403 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened 0e797cba1bb85ec7f7b6bb605c8e5361; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70379697, jitterRate=0.04873920977115631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:37,404 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for 0e797cba1bb85ec7f7b6bb605c8e5361: 2024-11-07T14:18:37,405 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361., pid=20, masterSystemTime=1730989117369 2024-11-07T14:18:37,409 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,409 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:18:37,410 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=0e797cba1bb85ec7f7b6bb605c8e5361, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:37,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T14:18:37,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 0e797cba1bb85ec7f7b6bb605c8e5361, server=69430dbfd73f,36593,1730989059679 in 197 msec 2024-11-07T14:18:37,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-07T14:18:37,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=0e797cba1bb85ec7f7b6bb605c8e5361, ASSIGN in 359 msec 2024-11-07T14:18:37,422 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:37,423 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989117422"}]},"ts":"1730989117422"} 2024-11-07T14:18:37,425 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:37,430 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:37,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 832 msec 2024-11-07T14:18:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T14:18:37,610 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-07T14:18:37,610 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:37,610 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:37,623 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:37,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:37,623 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-07T14:18:37,627 DEBUG [hconnection-0x11bedccb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:37,646 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48010, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:37,666 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:37,683 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T14:18:37,686 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T14:18:37,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668959 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:37,688 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T14:18:37,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668959 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:37,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628668959 2024-11-07T14:18:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:48010 deadline: 1730989118689, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:37,691 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:18:37,711 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T14:18:37,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668960 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:37,714 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T14:18:37,714 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:37,717 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:37,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668961 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:37,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628668961 2024-11-07T14:18:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:48010 deadline: 1730989118717, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:37,719 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:18:37,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:37,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668962 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:37,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4907802727628668962 2024-11-07T14:18:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:48010 deadline: 1730989118722, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:37,723 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:18:37,727 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:37,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668963 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:37,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4907802727628668963 2024-11-07T14:18:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:48010 deadline: 1730989118727, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:18:37,729 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:18:37,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:18:37,733 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ee57684 to 127.0.0.1:49795 2024-11-07T14:18:37,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:37,733 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:37,733 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:37,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:37,734 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x54c4b0b3 to 127.0.0.1:49795 2024-11-07T14:18:37,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=226 (was 225) - Thread LEAK? -, OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=5903 (was 5918) 2024-11-07T14:18:37,775 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=5903 2024-11-07T14:18:37,778 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27b8841a to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74e69b13 2024-11-07T14:18:37,783 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x27b8841a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f3296b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:37,785 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d538fd0 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16f7d771 2024-11-07T14:18:37,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758c09e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:37,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T14:18:37,816 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:37,816 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:37,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-07T14:18:37,818 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T14:18:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741845_1021 (size=426) 2024-11-07T14:18:37,834 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4f1f1bdcd9e70c414da10881c0097694, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:37,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741846_1022 (size=77) 2024-11-07T14:18:37,848 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:37,848 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 4f1f1bdcd9e70c414da10881c0097694, disabling compactions & flushes 2024-11-07T14:18:37,848 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:37,848 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:37,849 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. after waiting 0 ms 2024-11-07T14:18:37,849 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:37,849 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:37,849 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4f1f1bdcd9e70c414da10881c0097694: 2024-11-07T14:18:37,851 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:37,852 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730989117851"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989117851"}]},"ts":"1730989117851"} 2024-11-07T14:18:37,854 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:37,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:37,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989117855"}]},"ts":"1730989117855"} 2024-11-07T14:18:37,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:37,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=4f1f1bdcd9e70c414da10881c0097694, ASSIGN}] 2024-11-07T14:18:37,864 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=4f1f1bdcd9e70c414da10881c0097694, ASSIGN 2024-11-07T14:18:37,865 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=4f1f1bdcd9e70c414da10881c0097694, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:38,016 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=4f1f1bdcd9e70c414da10881c0097694, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:38,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 4f1f1bdcd9e70c414da10881c0097694, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:38,173 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:38,179 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:38,180 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => 4f1f1bdcd9e70c414da10881c0097694, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:38,181 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,181 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:38,181 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for 4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,181 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for 4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,184 INFO [StoreOpener-4f1f1bdcd9e70c414da10881c0097694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,188 INFO [StoreOpener-4f1f1bdcd9e70c414da10881c0097694-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f1f1bdcd9e70c414da10881c0097694 columnFamilyName testFamily 2024-11-07T14:18:38,188 DEBUG [StoreOpener-4f1f1bdcd9e70c414da10881c0097694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:38,189 INFO [StoreOpener-4f1f1bdcd9e70c414da10881c0097694-1 {}] regionserver.HStore(327): Store=4f1f1bdcd9e70c414da10881c0097694/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:38,190 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,191 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,195 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for 4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:18:38,198 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:38,199 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened 4f1f1bdcd9e70c414da10881c0097694; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69429364, jitterRate=0.034578144550323486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:38,200 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for 4f1f1bdcd9e70c414da10881c0097694: 2024-11-07T14:18:38,202 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694., pid=23, masterSystemTime=1730989118173 2024-11-07T14:18:38,205 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:38,205 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:18:38,206 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=4f1f1bdcd9e70c414da10881c0097694, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:38,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-07T14:18:38,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 4f1f1bdcd9e70c414da10881c0097694, server=69430dbfd73f,36593,1730989059679 in 189 msec 2024-11-07T14:18:38,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T14:18:38,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=4f1f1bdcd9e70c414da10881c0097694, ASSIGN in 349 msec 2024-11-07T14:18:38,216 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:38,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989118216"}]},"ts":"1730989118216"} 2024-11-07T14:18:38,218 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:38,222 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:38,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 421 msec 2024-11-07T14:18:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T14:18:38,822 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-07T14:18:38,822 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:38,822 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:38,827 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:38,828 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:38,828 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T14:18:38,831 DEBUG [hconnection-0x593cdaf3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:38,834 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48014, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:38,846 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:38,850 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:18:38,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:18:38,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T14:18:38,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T14:18:38,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:38,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T14:18:38,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-07T14:18:40,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:18:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:48014 deadline: 1730989121853, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:18:40,867 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:18:40,867 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668968 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:18:40,869 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2019 2024-11-07T14:18:40,870 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d538fd0 to 127.0.0.1:49795 2024-11-07T14:18:40,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:40,870 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:40,870 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:40,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:40,871 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x27b8841a to 127.0.0.1:49795 2024-11-07T14:18:40,883 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=227 (was 226) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=5898 (was 5903) 2024-11-07T14:18:40,895 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=5898 2024-11-07T14:18:40,897 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38a6b595 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@78a16f0a 2024-11-07T14:18:40,902 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x38a6b595 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ac5ab7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:40,904 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3420f2c4 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ad44d98 2024-11-07T14:18:40,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777cc10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:40,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T14:18:40,915 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:40,915 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:40,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-07T14:18:40,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T14:18:40,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741847_1023 (size=427) 2024-11-07T14:18:41,338 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6c03090e181c03a5de5603313432cd9a, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:41,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741848_1024 (size=78) 2024-11-07T14:18:41,348 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:41,349 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 6c03090e181c03a5de5603313432cd9a, disabling compactions & flushes 2024-11-07T14:18:41,349 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,349 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,349 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. after waiting 0 ms 2024-11-07T14:18:41,349 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,349 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,349 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6c03090e181c03a5de5603313432cd9a: 2024-11-07T14:18:41,351 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:41,351 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1730989121351"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989121351"}]},"ts":"1730989121351"} 2024-11-07T14:18:41,354 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:41,356 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:41,356 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989121356"}]},"ts":"1730989121356"} 2024-11-07T14:18:41,358 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:41,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6c03090e181c03a5de5603313432cd9a, ASSIGN}] 2024-11-07T14:18:41,365 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6c03090e181c03a5de5603313432cd9a, ASSIGN 2024-11-07T14:18:41,366 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6c03090e181c03a5de5603313432cd9a, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:41,517 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=6c03090e181c03a5de5603313432cd9a, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:41,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 6c03090e181c03a5de5603313432cd9a, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:41,674 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:41,690 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,690 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => 6c03090e181c03a5de5603313432cd9a, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:41,691 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,691 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:41,691 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for 6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,691 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for 6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,694 INFO [StoreOpener-6c03090e181c03a5de5603313432cd9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,696 INFO [StoreOpener-6c03090e181c03a5de5603313432cd9a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c03090e181c03a5de5603313432cd9a columnFamilyName testFamily 2024-11-07T14:18:41,697 DEBUG [StoreOpener-6c03090e181c03a5de5603313432cd9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:41,697 INFO [StoreOpener-6c03090e181c03a5de5603313432cd9a-1 {}] regionserver.HStore(327): Store=6c03090e181c03a5de5603313432cd9a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:41,700 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,700 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,701 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-07T14:18:41,701 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-07T14:18:41,704 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for 6c03090e181c03a5de5603313432cd9a 2024-11-07T14:18:41,708 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:41,709 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened 6c03090e181c03a5de5603313432cd9a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70665227, jitterRate=0.05299393832683563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:41,710 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for 6c03090e181c03a5de5603313432cd9a: 2024-11-07T14:18:41,711 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., pid=26, masterSystemTime=1730989121674 2024-11-07T14:18:41,714 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,714 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:18:41,715 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=6c03090e181c03a5de5603313432cd9a, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:41,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T14:18:41,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 6c03090e181c03a5de5603313432cd9a, server=69430dbfd73f,36593,1730989059679 in 201 msec 2024-11-07T14:18:41,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-07T14:18:41,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6c03090e181c03a5de5603313432cd9a, ASSIGN in 361 msec 2024-11-07T14:18:41,729 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:41,729 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989121729"}]},"ts":"1730989121729"} 2024-11-07T14:18:41,732 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:41,736 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:41,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 824 msec 2024-11-07T14:18:41,748 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:41411,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:18:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T14:18:41,921 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-07T14:18:41,921 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:41,921 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:41,927 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:41,928 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:41,928 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T14:18:41,931 DEBUG [hconnection-0x5a318754-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:41,934 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42686, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:41,946 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:41,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:18:41,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:18:42,341 ERROR [LeaseRenewer:jenkins@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:41411,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:18:42,881 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:18:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-07T14:18:43,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:18:44,955 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5009: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5009: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:18:44,956 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T14:18:44.955Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5009: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5009: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:18:44,956 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3007 2024-11-07T14:18:44,957 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3420f2c4 to 127.0.0.1:49795 2024-11-07T14:18:44,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:44,957 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:44,957 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:44,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:44,957 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38a6b595 to 127.0.0.1:49795 2024-11-07T14:18:44,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225 (was 227), OpenFileDescriptor=456 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 429), ProcessCount=11 (was 11), AvailableMemoryMB=5894 (was 5898) 2024-11-07T14:18:44,975 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=5894 2024-11-07T14:18:44,976 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1532bcb1 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4094772a 2024-11-07T14:18:44,981 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x1532bcb1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ecb2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:44,982 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c0e3423 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40b375cb 2024-11-07T14:18:44,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e49e58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:44,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T14:18:44,991 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:44,991 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:44,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-07T14:18:44,992 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T14:18:45,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741849_1025 (size=425) 2024-11-07T14:18:45,404 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b6f9ffe8865cf5861efa2412fd098b17, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:45,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741850_1026 (size=76) 2024-11-07T14:18:45,469 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:18:45,813 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:45,813 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing b6f9ffe8865cf5861efa2412fd098b17, disabling compactions & flushes 2024-11-07T14:18:45,813 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:45,813 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:45,813 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. after waiting 0 ms 2024-11-07T14:18:45,813 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:45,813 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:45,813 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for b6f9ffe8865cf5861efa2412fd098b17: 2024-11-07T14:18:45,815 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:45,815 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1730989125815"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989125815"}]},"ts":"1730989125815"} 2024-11-07T14:18:45,817 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:45,818 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:45,818 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989125818"}]},"ts":"1730989125818"} 2024-11-07T14:18:45,820 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:45,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b6f9ffe8865cf5861efa2412fd098b17, ASSIGN}] 2024-11-07T14:18:45,826 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b6f9ffe8865cf5861efa2412fd098b17, ASSIGN 2024-11-07T14:18:45,827 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b6f9ffe8865cf5861efa2412fd098b17, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:45,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=b6f9ffe8865cf5861efa2412fd098b17, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:45,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure b6f9ffe8865cf5861efa2412fd098b17, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T14:18:46,133 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:46,138 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:46,139 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => b6f9ffe8865cf5861efa2412fd098b17, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:46,139 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,139 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:46,139 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,139 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,141 INFO [StoreOpener-b6f9ffe8865cf5861efa2412fd098b17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,143 INFO [StoreOpener-b6f9ffe8865cf5861efa2412fd098b17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6f9ffe8865cf5861efa2412fd098b17 columnFamilyName testFamily 2024-11-07T14:18:46,143 DEBUG [StoreOpener-b6f9ffe8865cf5861efa2412fd098b17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:46,144 INFO [StoreOpener-b6f9ffe8865cf5861efa2412fd098b17-1 {}] regionserver.HStore(327): Store=b6f9ffe8865cf5861efa2412fd098b17/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:46,145 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,145 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,148 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:18:46,151 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:46,151 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened b6f9ffe8865cf5861efa2412fd098b17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63246875, jitterRate=-0.057548120617866516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:46,152 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for b6f9ffe8865cf5861efa2412fd098b17: 2024-11-07T14:18:46,153 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17., pid=29, masterSystemTime=1730989126133 2024-11-07T14:18:46,156 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:46,156 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:18:46,156 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=b6f9ffe8865cf5861efa2412fd098b17, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:46,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T14:18:46,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure b6f9ffe8865cf5861efa2412fd098b17, server=69430dbfd73f,36593,1730989059679 in 179 msec 2024-11-07T14:18:46,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T14:18:46,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b6f9ffe8865cf5861efa2412fd098b17, ASSIGN in 337 msec 2024-11-07T14:18:46,165 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:46,165 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989126165"}]},"ts":"1730989126165"} 2024-11-07T14:18:46,167 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:46,170 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:46,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.1820 sec 2024-11-07T14:18:47,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-07T14:18:47,704 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-07T14:18:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T14:18:47,997 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-07T14:18:47,997 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:47,997 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:48,002 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:48,002 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:48,002 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T14:18:48,004 DEBUG [hconnection-0x61cc78ab-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:48,006 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42694, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:48,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:48,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:18:48,019 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:18:48,021 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x1532bcb1 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:18:48,021 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x1532bcb1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:48,023 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42700, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:18:48,026 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:18:48,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668980 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:48,027 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:18:48,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668980 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:18:48,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T14:18:48,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T14:18:48,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:48,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T14:18:48,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:18:51,034 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3007 2024-11-07T14:18:51,034 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c0e3423 to 127.0.0.1:49795 2024-11-07T14:18:51,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:51,035 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:18:51,035 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668980 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:51,035 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:18:51,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:18:51,035 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1532bcb1 to 127.0.0.1:49795 2024-11-07T14:18:51,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628668980' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T14:18:51,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:42700 deadline: 1730989132034, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628668980' does not exist 2024-11-07T14:18:51,038 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:18:51,038 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 69430dbfd73f,36593,1730989059679 for closing scanner id = 4907802727628668980 for b6f9ffe8865cf5861efa2412fd098b17 of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T14:18:51,044 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=455 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 429), ProcessCount=11 (was 11), AvailableMemoryMB=5879 (was 5894) 2024-11-07T14:18:51,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=5879 2024-11-07T14:18:51,055 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0779338b to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a42092a 2024-11-07T14:18:51,058 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x0779338b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f8944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:51,060 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61f9733d to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e3cf76b 2024-11-07T14:18:51,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1097624c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:18:51,066 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:18:51,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-07T14:18:51,069 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:18:51,069 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:51,069 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-07T14:18:51,070 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:18:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T14:18:51,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741851_1027 (size=417) 2024-11-07T14:18:51,481 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ebd705d1230cccee05e70a4866f70c40, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:18:51,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741852_1028 (size=68) 2024-11-07T14:18:51,528 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:18:51,641 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:18:51,891 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:51,891 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ebd705d1230cccee05e70a4866f70c40, disabling compactions & flushes 2024-11-07T14:18:51,891 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:51,891 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:51,891 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. after waiting 0 ms 2024-11-07T14:18:51,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:51,892 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:51,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ebd705d1230cccee05e70a4866f70c40: 2024-11-07T14:18:51,894 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:18:51,894 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1730989131894"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989131894"}]},"ts":"1730989131894"} 2024-11-07T14:18:51,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:18:51,897 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:18:51,898 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989131897"}]},"ts":"1730989131897"} 2024-11-07T14:18:51,899 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:18:51,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=ebd705d1230cccee05e70a4866f70c40, ASSIGN}] 2024-11-07T14:18:51,904 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=ebd705d1230cccee05e70a4866f70c40, ASSIGN 2024-11-07T14:18:51,905 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=ebd705d1230cccee05e70a4866f70c40, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:18:52,056 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=ebd705d1230cccee05e70a4866f70c40, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:52,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure ebd705d1230cccee05e70a4866f70c40, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:18:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T14:18:52,211 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:18:52,216 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:52,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => ebd705d1230cccee05e70a4866f70c40, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:18:52,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:18:52,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,219 INFO [StoreOpener-ebd705d1230cccee05e70a4866f70c40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,221 INFO [StoreOpener-ebd705d1230cccee05e70a4866f70c40-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebd705d1230cccee05e70a4866f70c40 columnFamilyName testFamily 2024-11-07T14:18:52,221 DEBUG [StoreOpener-ebd705d1230cccee05e70a4866f70c40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:18:52,222 INFO [StoreOpener-ebd705d1230cccee05e70a4866f70c40-1 {}] regionserver.HStore(327): Store=ebd705d1230cccee05e70a4866f70c40/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:18:52,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:18:52,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:18:52,230 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened ebd705d1230cccee05e70a4866f70c40; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60870448, jitterRate=-0.09295964241027832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:18:52,231 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for ebd705d1230cccee05e70a4866f70c40: 2024-11-07T14:18:52,232 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40., pid=32, masterSystemTime=1730989132211 2024-11-07T14:18:52,234 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:52,234 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:18:52,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=ebd705d1230cccee05e70a4866f70c40, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:18:52,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T14:18:52,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure ebd705d1230cccee05e70a4866f70c40, server=69430dbfd73f,36593,1730989059679 in 179 msec 2024-11-07T14:18:52,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T14:18:52,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=ebd705d1230cccee05e70a4866f70c40, ASSIGN in 336 msec 2024-11-07T14:18:52,242 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:18:52,243 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989132242"}]},"ts":"1730989132242"} 2024-11-07T14:18:52,244 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:18:52,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:18:52,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 1.1810 sec 2024-11-07T14:18:53,553 INFO [master/69430dbfd73f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T14:18:53,553 INFO [master/69430dbfd73f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T14:18:53,705 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-07T14:18:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T14:18:54,075 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-07T14:18:54,075 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:18:54,075 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:54,080 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T14:18:54,080 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:18:54,080 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-07T14:18:54,082 DEBUG [hconnection-0x32dc7076-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:18:54,084 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46750, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:18:54,095 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:18:54,098 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:18:54,100 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668984 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:18:54,100 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T14:18:57,719 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:18:58,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-07T14:18:58,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:00,602 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6503 2024-11-07T14:19:00,602 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61f9733d to 127.0.0.1:49795 2024-11-07T14:19:00,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:00,602 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:00,602 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:00,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:00,602 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0779338b to 127.0.0.1:49795 2024-11-07T14:19:00,618 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 227) - Thread LEAK? -, OpenFileDescriptor=452 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 370) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5814 (was 5879) 2024-11-07T14:19:00,630 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=5814 2024-11-07T14:19:00,633 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52ab703e to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@415c0324 2024-11-07T14:19:00,639 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x52ab703e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfb5cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:00,641 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x309fe3e4 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@721e854c 2024-11-07T14:19:00,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3d21b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:00,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-07T14:19:00,657 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:00,657 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:00,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-07T14:19:00,659 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T14:19:00,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741853_1029 (size=421) 2024-11-07T14:19:00,698 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b93ccd845b07475bea616303fbd6a1a6, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:00,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741854_1030 (size=72) 2024-11-07T14:19:01,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:01,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing b93ccd845b07475bea616303fbd6a1a6, disabling compactions & flushes 2024-11-07T14:19:01,121 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. after waiting 0 ms 2024-11-07T14:19:01,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,121 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for b93ccd845b07475bea616303fbd6a1a6: 2024-11-07T14:19:01,123 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:01,124 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730989141123"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989141123"}]},"ts":"1730989141123"} 2024-11-07T14:19:01,131 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:01,134 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:01,134 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989141134"}]},"ts":"1730989141134"} 2024-11-07T14:19:01,137 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:19:01,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b93ccd845b07475bea616303fbd6a1a6, ASSIGN}] 2024-11-07T14:19:01,148 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b93ccd845b07475bea616303fbd6a1a6, ASSIGN 2024-11-07T14:19:01,153 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b93ccd845b07475bea616303fbd6a1a6, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:01,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=b93ccd845b07475bea616303fbd6a1a6, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:01,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure b93ccd845b07475bea616303fbd6a1a6, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:01,463 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:01,470 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,470 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => b93ccd845b07475bea616303fbd6a1a6, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:01,471 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,471 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:01,471 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,471 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,478 INFO [StoreOpener-b93ccd845b07475bea616303fbd6a1a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,481 INFO [StoreOpener-b93ccd845b07475bea616303fbd6a1a6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b93ccd845b07475bea616303fbd6a1a6 columnFamilyName testFamily 2024-11-07T14:19:01,481 DEBUG [StoreOpener-b93ccd845b07475bea616303fbd6a1a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:01,482 INFO [StoreOpener-b93ccd845b07475bea616303fbd6a1a6-1 {}] regionserver.HStore(327): Store=b93ccd845b07475bea616303fbd6a1a6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:01,484 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,484 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,488 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:19:01,492 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:01,493 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened b93ccd845b07475bea616303fbd6a1a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60472271, jitterRate=-0.098892942070961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:01,494 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for b93ccd845b07475bea616303fbd6a1a6: 2024-11-07T14:19:01,495 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6., pid=35, masterSystemTime=1730989141463 2024-11-07T14:19:01,500 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,500 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:19:01,501 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=b93ccd845b07475bea616303fbd6a1a6, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:01,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T14:19:01,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure b93ccd845b07475bea616303fbd6a1a6, server=69430dbfd73f,36593,1730989059679 in 196 msec 2024-11-07T14:19:01,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T14:19:01,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b93ccd845b07475bea616303fbd6a1a6, ASSIGN in 362 msec 2024-11-07T14:19:01,511 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:01,511 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989141511"}]},"ts":"1730989141511"} 2024-11-07T14:19:01,517 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:19:01,521 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:01,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 868 msec 2024-11-07T14:19:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T14:19:01,663 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-07T14:19:01,663 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:19:01,664 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:01,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T14:19:01,670 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:01,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-07T14:19:01,673 DEBUG [hconnection-0x10658da-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:01,684 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51946, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:01,704 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:01,708 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:19:01,714 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668988 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:19:01,714 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T14:19:03,215 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1507 2024-11-07T14:19:03,216 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x309fe3e4 to 127.0.0.1:49795 2024-11-07T14:19:03,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:03,216 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:03,216 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:03,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:03,216 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x52ab703e to 127.0.0.1:49795 2024-11-07T14:19:03,228 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=5804 (was 5814) 2024-11-07T14:19:03,240 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=5804 2024-11-07T14:19:03,242 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31a189bd to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@71a6b70f 2024-11-07T14:19:03,246 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x31a189bd {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5413f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:03,247 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c133953 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@22478921 2024-11-07T14:19:03,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1610b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:03,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-07T14:19:03,258 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:03,258 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:03,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-07T14:19:03,261 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T14:19:03,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741855_1031 (size=420) 2024-11-07T14:19:03,676 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9706565c662723050a81bf87f4213369, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:03,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741856_1032 (size=71) 2024-11-07T14:19:03,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:03,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 9706565c662723050a81bf87f4213369, disabling compactions & flushes 2024-11-07T14:19:03,723 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:03,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:03,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. after waiting 0 ms 2024-11-07T14:19:03,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:03,723 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:03,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9706565c662723050a81bf87f4213369: 2024-11-07T14:19:03,725 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:03,726 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730989143726"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989143726"}]},"ts":"1730989143726"} 2024-11-07T14:19:03,729 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:03,731 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:03,731 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989143731"}]},"ts":"1730989143731"} 2024-11-07T14:19:03,734 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:19:03,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=9706565c662723050a81bf87f4213369, ASSIGN}] 2024-11-07T14:19:03,743 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=9706565c662723050a81bf87f4213369, ASSIGN 2024-11-07T14:19:03,746 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=9706565c662723050a81bf87f4213369, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:03,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=9706565c662723050a81bf87f4213369, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:03,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 9706565c662723050a81bf87f4213369, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:04,061 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:04,067 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:04,067 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 9706565c662723050a81bf87f4213369, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:04,068 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,068 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:04,068 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,068 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,073 INFO [StoreOpener-9706565c662723050a81bf87f4213369-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,076 INFO [StoreOpener-9706565c662723050a81bf87f4213369-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9706565c662723050a81bf87f4213369 columnFamilyName testFamily 2024-11-07T14:19:04,076 DEBUG [StoreOpener-9706565c662723050a81bf87f4213369-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:04,077 INFO [StoreOpener-9706565c662723050a81bf87f4213369-1 {}] regionserver.HStore(327): Store=9706565c662723050a81bf87f4213369/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:04,080 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,081 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,085 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 9706565c662723050a81bf87f4213369 2024-11-07T14:19:04,090 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:04,091 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 9706565c662723050a81bf87f4213369; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65362899, jitterRate=-0.026016905903816223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:04,092 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 9706565c662723050a81bf87f4213369: 2024-11-07T14:19:04,093 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369., pid=38, masterSystemTime=1730989144061 2024-11-07T14:19:04,096 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:04,096 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:19:04,097 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=9706565c662723050a81bf87f4213369, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:04,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T14:19:04,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 9706565c662723050a81bf87f4213369, server=69430dbfd73f,36593,1730989059679 in 192 msec 2024-11-07T14:19:04,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T14:19:04,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=9706565c662723050a81bf87f4213369, ASSIGN in 361 msec 2024-11-07T14:19:04,107 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:04,107 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989144107"}]},"ts":"1730989144107"} 2024-11-07T14:19:04,110 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:19:04,114 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:04,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 859 msec 2024-11-07T14:19:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T14:19:04,267 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-07T14:19:04,267 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:19:04,267 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:04,274 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T14:19:04,274 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:04,274 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-07T14:19:04,280 DEBUG [hconnection-0x4f1c1663-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:04,283 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51960, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:04,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:04,306 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x236fb6a0 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4bfd4197 2024-11-07T14:19:04,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9267ce5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:04,320 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:19:04,327 DEBUG [hconnection-0x407b8e2c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:04,331 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51970, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:04,340 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:19:04,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668993 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:04,343 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:19:04,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668993 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:04,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T14:19:05,347 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T14:19:05.344Z, RpcRetryingCaller{globalStartTime=2024-11-07T14:19:04.343Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T14:19:05.344Z, RpcRetryingCaller{globalStartTime=2024-11-07T14:19:04.343Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:19:05,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1004 2024-11-07T14:19:05,348 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x236fb6a0 to 127.0.0.1:49795 2024-11-07T14:19:05,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:05,349 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c133953 to 127.0.0.1:49795 2024-11-07T14:19:05,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:05,349 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:05,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:05,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:05,349 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31a189bd to 127.0.0.1:49795 2024-11-07T14:19:05,378 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 231) - Thread LEAK? -, OpenFileDescriptor=461 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 391) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5760 (was 5804) 2024-11-07T14:19:05,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=5755 2024-11-07T14:19:05,419 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69b4b380 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58146068 2024-11-07T14:19:05,435 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x69b4b380 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2daed114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:05,437 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35eccb45 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@567c89db 2024-11-07T14:19:05,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a980827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:05,471 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:05,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T14:19:05,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:05,494 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:05,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:05,498 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-07T14:19:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T14:19:05,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741857_1033 (size=422) 2024-11-07T14:19:05,571 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a9b718113897f63098bd30ff08ddfe3c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741858_1034 (size=73) 2024-11-07T14:19:05,600 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:05,601 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing a9b718113897f63098bd30ff08ddfe3c, disabling compactions & flushes 2024-11-07T14:19:05,601 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,601 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,601 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. after waiting 0 ms 2024-11-07T14:19:05,601 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,601 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,601 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for a9b718113897f63098bd30ff08ddfe3c: 2024-11-07T14:19:05,603 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:05,604 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1730989145603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989145603"}]},"ts":"1730989145603"} 2024-11-07T14:19:05,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:05,607 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:05,607 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989145607"}]},"ts":"1730989145607"} 2024-11-07T14:19:05,610 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:19:05,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=a9b718113897f63098bd30ff08ddfe3c, ASSIGN}] 2024-11-07T14:19:05,622 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=a9b718113897f63098bd30ff08ddfe3c, ASSIGN 2024-11-07T14:19:05,623 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=a9b718113897f63098bd30ff08ddfe3c, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:05,707 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-07T14:19:05,707 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-07T14:19:05,774 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=a9b718113897f63098bd30ff08ddfe3c, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:05,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure a9b718113897f63098bd30ff08ddfe3c, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:05,846 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:19:05,929 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:05,936 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,937 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => a9b718113897f63098bd30ff08ddfe3c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:05,937 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,938 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:05,938 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,938 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,946 INFO [StoreOpener-a9b718113897f63098bd30ff08ddfe3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,949 INFO [StoreOpener-a9b718113897f63098bd30ff08ddfe3c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9b718113897f63098bd30ff08ddfe3c columnFamilyName testFamily 2024-11-07T14:19:05,949 DEBUG [StoreOpener-a9b718113897f63098bd30ff08ddfe3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:05,950 INFO [StoreOpener-a9b718113897f63098bd30ff08ddfe3c-1 {}] regionserver.HStore(327): Store=a9b718113897f63098bd30ff08ddfe3c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:05,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:19:05,962 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:05,963 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened a9b718113897f63098bd30ff08ddfe3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60074344, jitterRate=-0.10482251644134521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:05,964 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for a9b718113897f63098bd30ff08ddfe3c: 2024-11-07T14:19:05,966 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c., pid=41, masterSystemTime=1730989145929 2024-11-07T14:19:05,970 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=a9b718113897f63098bd30ff08ddfe3c, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:05,971 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,971 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:19:05,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T14:19:05,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure a9b718113897f63098bd30ff08ddfe3c, server=69430dbfd73f,36593,1730989059679 in 196 msec 2024-11-07T14:19:05,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T14:19:05,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=a9b718113897f63098bd30ff08ddfe3c, ASSIGN in 357 msec 2024-11-07T14:19:05,983 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:05,983 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989145983"}]},"ts":"1730989145983"} 2024-11-07T14:19:05,986 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:19:05,990 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:05,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 519 msec 2024-11-07T14:19:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T14:19:06,503 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-07T14:19:06,503 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:19:06,503 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:06,511 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T14:19:06,511 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:06,511 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T14:19:06,514 DEBUG [hconnection-0x5a92e979-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:06,515 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51978, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:06,529 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:06,532 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:19:06,542 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:19:06,542 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T14:19:06,824 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:19:06,977 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:19:08,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-07T14:19:08,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:08,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T14:19:08,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:08,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T14:19:08,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:11,709 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-07T14:19:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-07T14:19:13,196 DEBUG [master/69430dbfd73f:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T14:19:13,200 DEBUG [master/69430dbfd73f:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region f292f0dddc0193d61d9f927f4dc3af4c changed from -1.0 to 0.0, refreshing cache 2024-11-07T14:19:13,556 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:19:13,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51978 deadline: 1730989155541, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:19:13,557 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:19:13,558 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628668997 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:19:13,559 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7026 2024-11-07T14:19:13,559 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x35eccb45 to 127.0.0.1:49795 2024-11-07T14:19:13,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:13,559 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:13,560 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:13,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:13,560 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69b4b380 to 127.0.0.1:49795 2024-11-07T14:19:13,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=231 (was 236), OpenFileDescriptor=452 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5734 (was 5755) 2024-11-07T14:19:13,581 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=231, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=5734 2024-11-07T14:19:13,583 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76a0273d to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6031ad6e 2024-11-07T14:19:13,587 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x76a0273d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9e923d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:13,588 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3516d201 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ac4a55d 2024-11-07T14:19:13,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5daae6f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:13,594 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:13,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T14:19:13,597 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:13,597 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:13,597 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-07T14:19:13,598 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T14:19:13,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741859_1035 (size=428) 2024-11-07T14:19:14,010 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 93cfb54d75f976bc77e2ff0d4c799e55, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:14,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741860_1036 (size=79) 2024-11-07T14:19:14,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:14,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 93cfb54d75f976bc77e2ff0d4c799e55, disabling compactions & flushes 2024-11-07T14:19:14,025 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. after waiting 0 ms 2024-11-07T14:19:14,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,025 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 93cfb54d75f976bc77e2ff0d4c799e55: 2024-11-07T14:19:14,027 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:14,027 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1730989154027"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989154027"}]},"ts":"1730989154027"} 2024-11-07T14:19:14,030 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:14,031 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:14,031 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989154031"}]},"ts":"1730989154031"} 2024-11-07T14:19:14,033 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:19:14,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=93cfb54d75f976bc77e2ff0d4c799e55, ASSIGN}] 2024-11-07T14:19:14,038 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=93cfb54d75f976bc77e2ff0d4c799e55, ASSIGN 2024-11-07T14:19:14,040 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=93cfb54d75f976bc77e2ff0d4c799e55, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:14,190 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=93cfb54d75f976bc77e2ff0d4c799e55, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:14,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 93cfb54d75f976bc77e2ff0d4c799e55, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:14,346 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:14,352 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,352 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 93cfb54d75f976bc77e2ff0d4c799e55, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:14,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:14,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,358 INFO [StoreOpener-93cfb54d75f976bc77e2ff0d4c799e55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,360 INFO [StoreOpener-93cfb54d75f976bc77e2ff0d4c799e55-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93cfb54d75f976bc77e2ff0d4c799e55 columnFamilyName testFamily 2024-11-07T14:19:14,360 DEBUG [StoreOpener-93cfb54d75f976bc77e2ff0d4c799e55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:14,360 INFO [StoreOpener-93cfb54d75f976bc77e2ff0d4c799e55-1 {}] regionserver.HStore(327): Store=93cfb54d75f976bc77e2ff0d4c799e55/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:14,362 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,362 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,366 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:19:14,369 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:14,370 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 93cfb54d75f976bc77e2ff0d4c799e55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63255343, jitterRate=-0.057421937584877014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:14,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 93cfb54d75f976bc77e2ff0d4c799e55: 2024-11-07T14:19:14,372 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., pid=44, masterSystemTime=1730989154346 2024-11-07T14:19:14,375 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,375 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:19:14,376 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=93cfb54d75f976bc77e2ff0d4c799e55, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:14,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-07T14:19:14,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 93cfb54d75f976bc77e2ff0d4c799e55, server=69430dbfd73f,36593,1730989059679 in 184 msec 2024-11-07T14:19:14,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T14:19:14,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=93cfb54d75f976bc77e2ff0d4c799e55, ASSIGN in 344 msec 2024-11-07T14:19:14,385 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:14,385 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989154385"}]},"ts":"1730989154385"} 2024-11-07T14:19:14,387 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:19:14,390 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:14,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 797 msec 2024-11-07T14:19:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T14:19:14,601 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-07T14:19:14,602 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:19:14,602 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:14,607 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T14:19:14,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:14,608 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T14:19:14,611 DEBUG [hconnection-0x1f14834c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:14,612 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54984, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:14,624 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:14,627 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:19:14,627 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:19:14,629 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x76a0273d {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:19:14,629 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x76a0273d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:14,631 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54998, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:19:14,635 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:19:15,005 ERROR [LeaseRenewer:jenkins@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:41411,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:19:16,153 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:41411,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:19:17,710 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T14:19:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T14:19:18,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:19,892 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:19:20,646 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-07T14:19:20,646 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-07T14:19:20,646 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:19:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-07T14:19:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:19:21,662 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:19:23,634 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1972ms, rpcTimeout=1965ms 2024-11-07T14:19:23,634 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1972ms, rpcTimeout=1965ms 2024-11-07T14:19:23,634 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:19:23,635 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T14:19:20.647Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-07T14:19:23.634Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1972ms, rpcTimeout=1965ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1972ms, rpcTimeout=1965ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1972ms, rpcTimeout=1965ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T14:19:23,636 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-07T14:19:23,636 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3516d201 to 127.0.0.1:49795 2024-11-07T14:19:23,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:23,636 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:23,637 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:23,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:23,637 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76a0273d to 127.0.0.1:49795 2024-11-07T14:19:23,646 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229 (was 231), OpenFileDescriptor=448 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 452), ProcessCount=11 (was 11), AvailableMemoryMB=5720 (was 5734) 2024-11-07T14:19:23,655 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=5719 2024-11-07T14:19:23,656 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x04e0ca97 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69123919 2024-11-07T14:19:23,660 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x04e0ca97 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158c31c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:23,661 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21a68049 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c6218d1 2024-11-07T14:19:23,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed4cbb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:23,667 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:23,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-07T14:19:23,669 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:23,669 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:23,669 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-07T14:19:23,670 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T14:19:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741861_1037 (size=416) 2024-11-07T14:19:23,960 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668948 expired clientIPAndPort=172.17.0.2:52028, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:19:24,080 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c6d73ad91ef7cb6fc0537617e72d4e34, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:24,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741862_1038 (size=67) 2024-11-07T14:19:24,488 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:24,488 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing c6d73ad91ef7cb6fc0537617e72d4e34, disabling compactions & flushes 2024-11-07T14:19:24,488 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,489 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,489 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. after waiting 0 ms 2024-11-07T14:19:24,489 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,489 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,489 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for c6d73ad91ef7cb6fc0537617e72d4e34: 2024-11-07T14:19:24,490 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:24,490 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1730989164490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989164490"}]},"ts":"1730989164490"} 2024-11-07T14:19:24,492 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:24,493 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:24,493 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989164493"}]},"ts":"1730989164493"} 2024-11-07T14:19:24,495 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:19:24,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=c6d73ad91ef7cb6fc0537617e72d4e34, ASSIGN}] 2024-11-07T14:19:24,499 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=c6d73ad91ef7cb6fc0537617e72d4e34, ASSIGN 2024-11-07T14:19:24,500 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=c6d73ad91ef7cb6fc0537617e72d4e34, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:24,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=c6d73ad91ef7cb6fc0537617e72d4e34, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:24,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure c6d73ad91ef7cb6fc0537617e72d4e34, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T14:19:24,805 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:24,812 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => c6d73ad91ef7cb6fc0537617e72d4e34, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:24,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:24,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,817 INFO [StoreOpener-c6d73ad91ef7cb6fc0537617e72d4e34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,819 INFO [StoreOpener-c6d73ad91ef7cb6fc0537617e72d4e34-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d73ad91ef7cb6fc0537617e72d4e34 columnFamilyName testFamily 2024-11-07T14:19:24,819 DEBUG [StoreOpener-c6d73ad91ef7cb6fc0537617e72d4e34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:24,820 INFO [StoreOpener-c6d73ad91ef7cb6fc0537617e72d4e34-1 {}] regionserver.HStore(327): Store=c6d73ad91ef7cb6fc0537617e72d4e34/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:24,821 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,822 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,825 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:19:24,835 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:24,836 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened c6d73ad91ef7cb6fc0537617e72d4e34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69210781, jitterRate=0.031321004033088684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:24,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for c6d73ad91ef7cb6fc0537617e72d4e34: 2024-11-07T14:19:24,838 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34., pid=47, masterSystemTime=1730989164805 2024-11-07T14:19:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,841 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:19:24,841 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=c6d73ad91ef7cb6fc0537617e72d4e34, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:24,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T14:19:24,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure c6d73ad91ef7cb6fc0537617e72d4e34, server=69430dbfd73f,36593,1730989059679 in 190 msec 2024-11-07T14:19:24,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-07T14:19:24,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=c6d73ad91ef7cb6fc0537617e72d4e34, ASSIGN in 348 msec 2024-11-07T14:19:24,849 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:24,849 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989164849"}]},"ts":"1730989164849"} 2024-11-07T14:19:24,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:19:24,854 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:24,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.1870 sec 2024-11-07T14:19:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T14:19:26,675 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-07T14:19:26,675 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:19:26,676 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:26,681 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T14:19:26,681 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:26,681 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-07T14:19:26,684 DEBUG [hconnection-0x77de32a4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:26,685 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33598, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:26,694 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:26,698 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x254355d9 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@444da731 2024-11-07T14:19:26,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76500d72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:26,702 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:19:26,703 DEBUG [hconnection-0x5f0d58a6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:26,705 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33606, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:26,707 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:19:26,708 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669010 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:26,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:19:26,710 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669010 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:26,710 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T14:19:28,163 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:19:28,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-07T14:19:28,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:29,712 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-07T14:19:30,315 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:19:30,968 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668950 expired clientIPAndPort=172.17.0.2:52028, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:19:32,714 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T14:19:32.714Z, RpcRetryingCaller{globalStartTime=2024-11-07T14:19:26.709Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T14:19:32.714Z, RpcRetryingCaller{globalStartTime=2024-11-07T14:19:26.709Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:19:32,715 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6005 2024-11-07T14:19:32,715 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x254355d9 to 127.0.0.1:49795 2024-11-07T14:19:32,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:32,715 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21a68049 to 127.0.0.1:49795 2024-11-07T14:19:32,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:32,716 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:32,716 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:32,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:32,716 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x04e0ca97 to 127.0.0.1:49795 2024-11-07T14:19:32,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=235 (was 229) - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=405 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=5628 (was 5719) 2024-11-07T14:19:32,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=235, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=405, ProcessCount=11, AvailableMemoryMB=5628 2024-11-07T14:19:32,746 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e36da37 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2327800d 2024-11-07T14:19:32,753 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x5e36da37 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a3e340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:32,754 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07f7534f to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4058adfc 2024-11-07T14:19:32,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d730a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:32,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T14:19:32,792 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:32,793 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:32,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-07T14:19:32,794 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T14:19:32,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741863_1039 (size=432) 2024-11-07T14:19:32,819 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d32a9e0dc85dc68730667345f4bf823f, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:32,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741864_1040 (size=83) 2024-11-07T14:19:32,862 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:32,862 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing d32a9e0dc85dc68730667345f4bf823f, disabling compactions & flushes 2024-11-07T14:19:32,862 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:32,862 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:32,862 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. after waiting 0 ms 2024-11-07T14:19:32,862 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:32,862 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:32,862 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for d32a9e0dc85dc68730667345f4bf823f: 2024-11-07T14:19:32,864 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:32,864 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1730989172864"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989172864"}]},"ts":"1730989172864"} 2024-11-07T14:19:32,867 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:32,869 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:32,869 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989172869"}]},"ts":"1730989172869"} 2024-11-07T14:19:32,873 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T14:19:32,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=d32a9e0dc85dc68730667345f4bf823f, ASSIGN}] 2024-11-07T14:19:32,882 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=d32a9e0dc85dc68730667345f4bf823f, ASSIGN 2024-11-07T14:19:32,883 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=d32a9e0dc85dc68730667345f4bf823f, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:33,033 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=d32a9e0dc85dc68730667345f4bf823f, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:33,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure d32a9e0dc85dc68730667345f4bf823f, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:33,190 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:33,197 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:33,197 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => d32a9e0dc85dc68730667345f4bf823f, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:33,197 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,198 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:33,198 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,198 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,200 INFO [StoreOpener-d32a9e0dc85dc68730667345f4bf823f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,203 INFO [StoreOpener-d32a9e0dc85dc68730667345f4bf823f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32a9e0dc85dc68730667345f4bf823f columnFamilyName testFamily 2024-11-07T14:19:33,203 DEBUG [StoreOpener-d32a9e0dc85dc68730667345f4bf823f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:33,204 INFO [StoreOpener-d32a9e0dc85dc68730667345f4bf823f-1 {}] regionserver.HStore(327): Store=d32a9e0dc85dc68730667345f4bf823f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:33,205 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,206 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,209 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:19:33,211 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:19:33,213 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:33,219 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened d32a9e0dc85dc68730667345f4bf823f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60857527, jitterRate=-0.09315218031406403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:33,220 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for d32a9e0dc85dc68730667345f4bf823f: 2024-11-07T14:19:33,221 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., pid=50, masterSystemTime=1730989173190 2024-11-07T14:19:33,224 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:33,224 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:19:33,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=d32a9e0dc85dc68730667345f4bf823f, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:33,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-07T14:19:33,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure d32a9e0dc85dc68730667345f4bf823f, server=69430dbfd73f,36593,1730989059679 in 191 msec 2024-11-07T14:19:33,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T14:19:33,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=d32a9e0dc85dc68730667345f4bf823f, ASSIGN in 353 msec 2024-11-07T14:19:33,239 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:33,239 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989173239"}]},"ts":"1730989173239"} 2024-11-07T14:19:33,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T14:19:33,246 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:33,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 456 msec 2024-11-07T14:19:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T14:19:33,798 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-07T14:19:33,798 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T14:19:33,798 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:33,803 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T14:19:33,803 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:33,803 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T14:19:33,805 DEBUG [hconnection-0x1ecea241-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:33,807 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59504, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:33,818 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:33,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:19:33,822 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:19:33,823 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x5e36da37 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:19:33,824 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x5e36da37 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:33,825 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59508, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:19:33,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:19:34,834 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-07T14:19:34,834 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-07T14:19:34,834 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:19:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-07T14:19:35,713 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T14:19:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:19:35,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:19:36,824 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:19:36,825 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=984ms, rpcTimeout=980ms 2024-11-07T14:19:36,825 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=984ms, rpcTimeout=980ms 2024-11-07T14:19:36,825 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:19:36,826 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T14:19:34.835Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-07T14:19:36.825Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=984ms, rpcTimeout=980ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=984ms, rpcTimeout=980ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=984ms, rpcTimeout=980ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T14:19:36,827 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2024-11-07T14:19:36,827 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x07f7534f to 127.0.0.1:49795 2024-11-07T14:19:36,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:36,827 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:36,828 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:36,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:36,828 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e36da37 to 127.0.0.1:49795 2024-11-07T14:19:36,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 235), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 405), ProcessCount=11 (was 11), AvailableMemoryMB=5689 (was 5628) - AvailableMemoryMB LEAK? - 2024-11-07T14:19:36,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=5688 2024-11-07T14:19:36,852 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x244ca45b to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3141f6bd 2024-11-07T14:19:36,859 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x244ca45b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22967618, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:36,861 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58e07d29 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69ce58dd 2024-11-07T14:19:36,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286157c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:36,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T14:19:36,881 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:36,881 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:36,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-07T14:19:36,882 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T14:19:36,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741865_1041 (size=436) 2024-11-07T14:19:36,900 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6db8379cd8b72d893208d9bc43753e66, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:36,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741866_1042 (size=87) 2024-11-07T14:19:36,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:36,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 6db8379cd8b72d893208d9bc43753e66, disabling compactions & flushes 2024-11-07T14:19:36,911 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:36,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:36,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. after waiting 0 ms 2024-11-07T14:19:36,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:36,911 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:36,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6db8379cd8b72d893208d9bc43753e66: 2024-11-07T14:19:36,913 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:36,913 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1730989176913"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989176913"}]},"ts":"1730989176913"} 2024-11-07T14:19:36,917 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:36,918 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:36,918 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989176918"}]},"ts":"1730989176918"} 2024-11-07T14:19:36,920 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:19:36,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6db8379cd8b72d893208d9bc43753e66, ASSIGN}] 2024-11-07T14:19:36,929 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6db8379cd8b72d893208d9bc43753e66, ASSIGN 2024-11-07T14:19:36,930 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6db8379cd8b72d893208d9bc43753e66, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:37,073 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668955 expired clientIPAndPort=172.17.0.2:50796, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:19:37,081 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=6db8379cd8b72d893208d9bc43753e66, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:37,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 6db8379cd8b72d893208d9bc43753e66, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:37,236 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:37,242 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:37,242 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => 6db8379cd8b72d893208d9bc43753e66, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:37,243 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,243 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:37,243 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for 6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,243 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for 6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,246 INFO [StoreOpener-6db8379cd8b72d893208d9bc43753e66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,248 INFO [StoreOpener-6db8379cd8b72d893208d9bc43753e66-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6db8379cd8b72d893208d9bc43753e66 columnFamilyName testFamily 2024-11-07T14:19:37,248 DEBUG [StoreOpener-6db8379cd8b72d893208d9bc43753e66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:37,250 INFO [StoreOpener-6db8379cd8b72d893208d9bc43753e66-1 {}] regionserver.HStore(327): Store=6db8379cd8b72d893208d9bc43753e66/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:37,251 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,252 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,256 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for 6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:19:37,259 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:37,260 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened 6db8379cd8b72d893208d9bc43753e66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72214510, jitterRate=0.07608005404472351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:37,261 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for 6db8379cd8b72d893208d9bc43753e66: 2024-11-07T14:19:37,262 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66., pid=53, masterSystemTime=1730989177236 2024-11-07T14:19:37,266 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:37,266 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:19:37,266 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=6db8379cd8b72d893208d9bc43753e66, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:37,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T14:19:37,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 6db8379cd8b72d893208d9bc43753e66, server=69430dbfd73f,36593,1730989059679 in 184 msec 2024-11-07T14:19:37,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-07T14:19:37,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6db8379cd8b72d893208d9bc43753e66, ASSIGN in 342 msec 2024-11-07T14:19:37,273 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:37,273 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989177273"}]},"ts":"1730989177273"} 2024-11-07T14:19:37,275 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:19:37,278 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:37,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 400 msec 2024-11-07T14:19:37,343 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:19:37,774 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668959 expired clientIPAndPort=172.17.0.2:48010, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:19:37,775 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668961 expired clientIPAndPort=172.17.0.2:48010, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:19:37,775 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668962 expired clientIPAndPort=172.17.0.2:48010, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:19:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T14:19:37,885 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-07T14:19:37,885 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:19:37,885 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:37,889 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T14:19:37,889 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:37,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-07T14:19:37,892 DEBUG [hconnection-0x7417bd36-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:37,893 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59510, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:37,902 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:37,906 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T14:19:37,906 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:19:37,908 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x244ca45b {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:19:37,908 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x244ca45b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:37,909 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59526, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:19:37,912 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T14:19:37,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669022 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:37,913 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T14:19:37,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669022 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:37,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628669022 2024-11-07T14:19:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:59526 deadline: 1730989180912, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:19:37,915 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669022 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:19:37,916 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T14:19:37,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669023 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:37,916 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T14:19:37,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:19:37,918 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:19:37,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669024 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:37,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628669024 2024-11-07T14:19:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:59526 deadline: 1730989180917, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:19:37,920 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669024 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:19:37,921 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:19:37,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669025 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:37,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4907802727628669025 2024-11-07T14:19:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:59526 deadline: 1730989180921, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:19:37,923 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669025 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:19:37,925 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:19:37,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669026 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:37,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4907802727628669026 2024-11-07T14:19:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:59526 deadline: 1730989180924, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:19:37,926 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669026 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:19:37,927 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:19:37,928 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58e07d29 to 127.0.0.1:49795 2024-11-07T14:19:37,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:37,928 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:37,928 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:37,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:37,928 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x244ca45b to 127.0.0.1:49795 2024-11-07T14:19:37,941 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232 (was 231) - Thread LEAK? -, OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=5686 (was 5688) 2024-11-07T14:19:37,957 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=232, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=5680 2024-11-07T14:19:37,959 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b457d27 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d6358fa 2024-11-07T14:19:37,963 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x5b457d27 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332bbda6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:37,964 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x778d21a2 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ff6d5d3 2024-11-07T14:19:37,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4faa3b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:37,980 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:37,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T14:19:37,983 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:37,983 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:37,984 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-07T14:19:37,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T14:19:38,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741867_1043 (size=422) 2024-11-07T14:19:38,011 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8e7403a5d1cd160a13036a66684da366, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:38,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741868_1044 (size=73) 2024-11-07T14:19:38,034 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:38,034 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 8e7403a5d1cd160a13036a66684da366, disabling compactions & flushes 2024-11-07T14:19:38,035 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,035 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,035 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. after waiting 0 ms 2024-11-07T14:19:38,035 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,035 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,035 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8e7403a5d1cd160a13036a66684da366: 2024-11-07T14:19:38,037 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:38,037 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1730989178037"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989178037"}]},"ts":"1730989178037"} 2024-11-07T14:19:38,039 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:38,040 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:38,041 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989178040"}]},"ts":"1730989178040"} 2024-11-07T14:19:38,043 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:19:38,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=8e7403a5d1cd160a13036a66684da366, ASSIGN}] 2024-11-07T14:19:38,048 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=8e7403a5d1cd160a13036a66684da366, ASSIGN 2024-11-07T14:19:38,049 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=8e7403a5d1cd160a13036a66684da366, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:38,200 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=8e7403a5d1cd160a13036a66684da366, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:38,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 8e7403a5d1cd160a13036a66684da366, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:38,360 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:38,366 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,366 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => 8e7403a5d1cd160a13036a66684da366, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:38,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:38,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for 8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for 8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,369 INFO [StoreOpener-8e7403a5d1cd160a13036a66684da366-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,371 INFO [StoreOpener-8e7403a5d1cd160a13036a66684da366-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e7403a5d1cd160a13036a66684da366 columnFamilyName testFamily 2024-11-07T14:19:38,371 DEBUG [StoreOpener-8e7403a5d1cd160a13036a66684da366-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:38,372 INFO [StoreOpener-8e7403a5d1cd160a13036a66684da366-1 {}] regionserver.HStore(327): Store=8e7403a5d1cd160a13036a66684da366/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:38,373 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for 8e7403a5d1cd160a13036a66684da366 2024-11-07T14:19:38,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:38,382 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened 8e7403a5d1cd160a13036a66684da366; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60623110, jitterRate=-0.09664526581764221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:38,383 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for 8e7403a5d1cd160a13036a66684da366: 2024-11-07T14:19:38,385 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., pid=56, masterSystemTime=1730989178360 2024-11-07T14:19:38,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,387 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:19:38,388 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=8e7403a5d1cd160a13036a66684da366, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:38,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-07T14:19:38,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 8e7403a5d1cd160a13036a66684da366, server=69430dbfd73f,36593,1730989059679 in 186 msec 2024-11-07T14:19:38,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T14:19:38,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=8e7403a5d1cd160a13036a66684da366, ASSIGN in 344 msec 2024-11-07T14:19:38,394 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:38,394 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989178394"}]},"ts":"1730989178394"} 2024-11-07T14:19:38,396 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:19:38,399 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:38,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 419 msec 2024-11-07T14:19:38,701 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:19:38,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T14:19:38,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:38,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T14:19:38,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:38,948 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T14:19:38,948 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T14:19:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T14:19:38,988 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-07T14:19:38,989 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:19:38,989 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:38,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T14:19:38,994 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:38,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T14:19:38,996 DEBUG [hconnection-0x31588fec-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:38,998 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59536, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:39,008 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:39,011 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:19:39,012 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:19:40,878 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668968 expired clientIPAndPort=172.17.0.2:48014, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:19:41,715 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-07T14:19:41,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-07T14:19:43,480 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668973 expired clientIPAndPort=172.17.0.2:42686, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:19:43,869 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:19:45,482 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668975 expired clientIPAndPort=172.17.0.2:42686, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:19:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-07T14:19:46,029 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:19:48,015 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:19:48,015 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T14:19:48.015Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:19:48,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2024-11-07T14:19:48,017 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x778d21a2 to 127.0.0.1:49795 2024-11-07T14:19:48,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:48,017 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:19:48,017 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:19:48,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:19:48,018 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b457d27 to 127.0.0.1:49795 2024-11-07T14:19:48,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=231 (was 232), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 404) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5653 (was 5680) 2024-11-07T14:19:48,035 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=5653 2024-11-07T14:19:48,037 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d742996 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@286d8c79 2024-11-07T14:19:48,041 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x6d742996 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5883a116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:48,042 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1e1232dc to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@210b2727 2024-11-07T14:19:48,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7f8209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:19:48,047 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:19:48,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T14:19:48,049 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:19:48,049 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:48,049 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-07T14:19:48,050 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:19:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T14:19:48,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741869_1045 (size=420) 2024-11-07T14:19:48,461 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d04a9d74f6b94963a147e616bf79af65, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:19:48,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741870_1046 (size=71) 2024-11-07T14:19:48,869 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:48,870 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing d04a9d74f6b94963a147e616bf79af65, disabling compactions & flushes 2024-11-07T14:19:48,870 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:48,870 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:48,870 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. after waiting 0 ms 2024-11-07T14:19:48,870 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:48,870 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:48,870 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for d04a9d74f6b94963a147e616bf79af65: 2024-11-07T14:19:48,871 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:19:48,871 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730989188871"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989188871"}]},"ts":"1730989188871"} 2024-11-07T14:19:48,873 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:19:48,874 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:19:48,874 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989188874"}]},"ts":"1730989188874"} 2024-11-07T14:19:48,876 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:19:48,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d04a9d74f6b94963a147e616bf79af65, ASSIGN}] 2024-11-07T14:19:48,881 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d04a9d74f6b94963a147e616bf79af65, ASSIGN 2024-11-07T14:19:48,882 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d04a9d74f6b94963a147e616bf79af65, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:19:49,032 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=d04a9d74f6b94963a147e616bf79af65, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:49,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure d04a9d74f6b94963a147e616bf79af65, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:19:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T14:19:49,186 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:19:49,192 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:49,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => d04a9d74f6b94963a147e616bf79af65, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:19:49,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:19:49,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,195 INFO [StoreOpener-d04a9d74f6b94963a147e616bf79af65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,197 INFO [StoreOpener-d04a9d74f6b94963a147e616bf79af65-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d04a9d74f6b94963a147e616bf79af65 columnFamilyName testFamily 2024-11-07T14:19:49,197 DEBUG [StoreOpener-d04a9d74f6b94963a147e616bf79af65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:19:49,197 INFO [StoreOpener-d04a9d74f6b94963a147e616bf79af65-1 {}] regionserver.HStore(327): Store=d04a9d74f6b94963a147e616bf79af65/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:19:49,198 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:19:49,204 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:19:49,204 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened d04a9d74f6b94963a147e616bf79af65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75333623, jitterRate=0.12255845963954926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:19:49,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for d04a9d74f6b94963a147e616bf79af65: 2024-11-07T14:19:49,206 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65., pid=59, masterSystemTime=1730989189186 2024-11-07T14:19:49,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:49,208 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:19:49,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=d04a9d74f6b94963a147e616bf79af65, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:19:49,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T14:19:49,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure d04a9d74f6b94963a147e616bf79af65, server=69430dbfd73f,36593,1730989059679 in 176 msec 2024-11-07T14:19:49,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T14:19:49,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=d04a9d74f6b94963a147e616bf79af65, ASSIGN in 332 msec 2024-11-07T14:19:49,214 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:19:49,214 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989189214"}]},"ts":"1730989189214"} 2024-11-07T14:19:49,215 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:19:49,218 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:19:49,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 1.1710 sec 2024-11-07T14:19:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T14:19:51,054 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-07T14:19:51,054 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:19:51,054 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:51,057 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T14:19:51,058 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:19:51,058 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T14:19:51,060 DEBUG [hconnection-0x4f0a1141-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:51,061 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49984, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:51,068 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:19:51,071 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:19:51,071 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:19:51,072 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x6d742996 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:19:51,072 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x6d742996 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:19:51,073 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49996, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:19:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:19:51,076 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:19:51,076 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669038 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:51,077 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:19:51,077 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669038 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:19:51,077 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T14:19:51,587 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668980 expired clientIPAndPort=172.17.0.2:42700, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:19:52,530 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:19:53,678 ERROR [LeaseRenewer:jenkins@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:41411,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:19:53,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-07T14:19:54,750 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:19:54,836 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:41411,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:19:58,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T14:19:58,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:00,085 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9007 2024-11-07T14:20:00,085 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1e1232dc to 127.0.0.1:49795 2024-11-07T14:20:00,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:00,085 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669038 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:20:00,085 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:00,085 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:00,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:00,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628669038' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T14:20:00,085 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d742996 to 127.0.0.1:49795 2024-11-07T14:20:00,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:49996 deadline: 1730989206085, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628669038' does not exist 2024-11-07T14:20:00,086 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 69430dbfd73f,36593,1730989059679 for closing scanner id = 4907802727628669038 for d04a9d74f6b94963a147e616bf79af65 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T14:20:00,089 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:00,096 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=226 (was 231), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 411), ProcessCount=11 (was 11), AvailableMemoryMB=5613 (was 5653) 2024-11-07T14:20:00,107 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=5613 2024-11-07T14:20:00,108 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x284b5d30 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ea9df66 2024-11-07T14:20:00,114 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x284b5d30 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6aa8b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:00,115 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ca03b9b to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7bb9b00 2024-11-07T14:20:00,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756bf41a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:00,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T14:20:00,123 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:00,123 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:00,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-07T14:20:00,124 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T14:20:00,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741871_1047 (size=431) 2024-11-07T14:20:00,151 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6fbbea4d395a3aec7237b0e5e1ceaae5, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:00,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741872_1048 (size=82) 2024-11-07T14:20:00,163 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:00,163 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 6fbbea4d395a3aec7237b0e5e1ceaae5, disabling compactions & flushes 2024-11-07T14:20:00,163 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,163 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,163 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. after waiting 0 ms 2024-11-07T14:20:00,163 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,163 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,163 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6fbbea4d395a3aec7237b0e5e1ceaae5: 2024-11-07T14:20:00,164 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:00,165 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1730989200164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989200164"}]},"ts":"1730989200164"} 2024-11-07T14:20:00,166 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:00,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:00,167 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989200167"}]},"ts":"1730989200167"} 2024-11-07T14:20:00,169 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:00,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=6fbbea4d395a3aec7237b0e5e1ceaae5, ASSIGN}] 2024-11-07T14:20:00,174 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=6fbbea4d395a3aec7237b0e5e1ceaae5, ASSIGN 2024-11-07T14:20:00,175 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=6fbbea4d395a3aec7237b0e5e1ceaae5, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:00,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=6fbbea4d395a3aec7237b0e5e1ceaae5, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:00,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 6fbbea4d395a3aec7237b0e5e1ceaae5, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:00,480 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:00,488 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,488 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 6fbbea4d395a3aec7237b0e5e1ceaae5, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:00,489 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,489 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:00,489 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,489 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,497 INFO [StoreOpener-6fbbea4d395a3aec7237b0e5e1ceaae5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,499 INFO [StoreOpener-6fbbea4d395a3aec7237b0e5e1ceaae5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbbea4d395a3aec7237b0e5e1ceaae5 columnFamilyName testFamily 2024-11-07T14:20:00,500 DEBUG [StoreOpener-6fbbea4d395a3aec7237b0e5e1ceaae5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:00,500 INFO [StoreOpener-6fbbea4d395a3aec7237b0e5e1ceaae5-1 {}] regionserver.HStore(327): Store=6fbbea4d395a3aec7237b0e5e1ceaae5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:00,502 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,503 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,506 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:20:00,521 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:00,522 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 6fbbea4d395a3aec7237b0e5e1ceaae5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72802717, jitterRate=0.08484502136707306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:00,523 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 6fbbea4d395a3aec7237b0e5e1ceaae5: 2024-11-07T14:20:00,524 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5., pid=62, masterSystemTime=1730989200480 2024-11-07T14:20:00,526 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,526 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:20:00,527 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=6fbbea4d395a3aec7237b0e5e1ceaae5, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:00,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T14:20:00,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 6fbbea4d395a3aec7237b0e5e1ceaae5, server=69430dbfd73f,36593,1730989059679 in 200 msec 2024-11-07T14:20:00,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T14:20:00,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=6fbbea4d395a3aec7237b0e5e1ceaae5, ASSIGN in 357 msec 2024-11-07T14:20:00,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:00,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989200533"}]},"ts":"1730989200533"} 2024-11-07T14:20:00,535 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:00,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:00,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 418 msec 2024-11-07T14:20:00,578 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T14:20:01,130 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-07T14:20:01,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:01,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:01,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:01,136 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:01,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-07T14:20:01,139 DEBUG [hconnection-0x79e1ce27-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:01,141 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47306, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:01,152 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:01,155 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T14:20:01,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T14:20:01,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669042 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:01,158 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T14:20:01,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669042 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:01,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628669042 2024-11-07T14:20:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:47306 deadline: 1730989204159, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:20:01,160 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:20:01,162 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T14:20:01,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669043 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:01,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T14:20:01,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:20:01,165 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:20:01,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669044 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:01,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4907802727628669044 2024-11-07T14:20:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:47306 deadline: 1730989204165, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:20:01,167 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:20:01,169 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:20:01,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669045 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:01,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4907802727628669045 2024-11-07T14:20:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:47306 deadline: 1730989204170, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:20:01,172 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:20:01,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:20:01,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669046 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:01,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4907802727628669046 2024-11-07T14:20:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:47306 deadline: 1730989204174, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T14:20:01,175 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:20:01,177 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T14:20:01,180 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ca03b9b to 127.0.0.1:49795 2024-11-07T14:20:01,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:01,180 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:01,180 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:01,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:01,180 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x284b5d30 to 127.0.0.1:49795 2024-11-07T14:20:01,192 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=234 (was 226) - Thread LEAK? -, OpenFileDescriptor=450 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5604 (was 5613) 2024-11-07T14:20:01,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=5604 2024-11-07T14:20:01,204 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25fa06c8 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@cf63e98 2024-11-07T14:20:01,208 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x25fa06c8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57255ee3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:01,209 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d805393 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d82e7e8 2024-11-07T14:20:01,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483b7ad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:01,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T14:20:01,218 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:01,218 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:01,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-07T14:20:01,219 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T14:20:01,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741873_1049 (size=425) 2024-11-07T14:20:01,233 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9ec939c454be0bf86a25c1d58e445783, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:01,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741874_1050 (size=76) 2024-11-07T14:20:01,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:01,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 9ec939c454be0bf86a25c1d58e445783, disabling compactions & flushes 2024-11-07T14:20:01,247 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. after waiting 0 ms 2024-11-07T14:20:01,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,247 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9ec939c454be0bf86a25c1d58e445783: 2024-11-07T14:20:01,248 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:01,248 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1730989201248"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989201248"}]},"ts":"1730989201248"} 2024-11-07T14:20:01,250 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:01,251 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:01,252 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989201251"}]},"ts":"1730989201251"} 2024-11-07T14:20:01,253 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:01,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=9ec939c454be0bf86a25c1d58e445783, ASSIGN}] 2024-11-07T14:20:01,259 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=9ec939c454be0bf86a25c1d58e445783, ASSIGN 2024-11-07T14:20:01,259 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=9ec939c454be0bf86a25c1d58e445783, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:01,410 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=9ec939c454be0bf86a25c1d58e445783, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:01,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure 9ec939c454be0bf86a25c1d58e445783, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:01,566 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:01,572 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,572 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => 9ec939c454be0bf86a25c1d58e445783, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:01,572 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true 9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,572 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:01,573 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for 9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,573 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for 9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,575 INFO [StoreOpener-9ec939c454be0bf86a25c1d58e445783-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,577 INFO [StoreOpener-9ec939c454be0bf86a25c1d58e445783-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ec939c454be0bf86a25c1d58e445783 columnFamilyName testFamily 2024-11-07T14:20:01,577 DEBUG [StoreOpener-9ec939c454be0bf86a25c1d58e445783-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:01,578 INFO [StoreOpener-9ec939c454be0bf86a25c1d58e445783-1 {}] regionserver.HStore(327): Store=9ec939c454be0bf86a25c1d58e445783/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:01,579 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,579 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,583 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for 9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:20:01,586 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:01,587 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened 9ec939c454be0bf86a25c1d58e445783; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60645148, jitterRate=-0.0963168740272522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:01,588 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for 9ec939c454be0bf86a25c1d58e445783: 2024-11-07T14:20:01,589 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783., pid=65, masterSystemTime=1730989201566 2024-11-07T14:20:01,591 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,591 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:20:01,592 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=9ec939c454be0bf86a25c1d58e445783, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:01,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T14:20:01,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure 9ec939c454be0bf86a25c1d58e445783, server=69430dbfd73f,36593,1730989059679 in 181 msec 2024-11-07T14:20:01,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T14:20:01,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=9ec939c454be0bf86a25c1d58e445783, ASSIGN in 338 msec 2024-11-07T14:20:01,599 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:01,599 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989201599"}]},"ts":"1730989201599"} 2024-11-07T14:20:01,601 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:01,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:01,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 389 msec 2024-11-07T14:20:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T14:20:02,225 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-07T14:20:02,225 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:02,225 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:02,249 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:02,249 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:02,249 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T14:20:02,252 DEBUG [hconnection-0x635e7e5e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:02,253 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47322, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:02,265 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:02,274 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:02,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:20:02,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T14:20:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-07T14:20:04,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:20:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:47322 deadline: 1730989205278, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:20:04,292 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:20:04,293 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669051 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:20:04,294 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2019 2024-11-07T14:20:04,295 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d805393 to 127.0.0.1:49795 2024-11-07T14:20:04,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:04,295 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:04,296 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:04,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:04,296 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25fa06c8 to 127.0.0.1:49795 2024-11-07T14:20:04,309 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234 (was 234), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=5599 (was 5604) 2024-11-07T14:20:04,322 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=234, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=5591 2024-11-07T14:20:04,324 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f7b8ba2 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5ab10571 2024-11-07T14:20:04,328 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x4f7b8ba2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295d7787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:04,329 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1cef9240 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@197eeed6 2024-11-07T14:20:04,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4b40c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:04,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T14:20:04,341 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:04,341 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:04,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-07T14:20:04,342 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T14:20:04,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741875_1051 (size=426) 2024-11-07T14:20:04,362 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7ebdf008998e34da6a8bfd565c333c77, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:04,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741876_1052 (size=77) 2024-11-07T14:20:04,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:04,774 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 7ebdf008998e34da6a8bfd565c333c77, disabling compactions & flushes 2024-11-07T14:20:04,774 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:04,774 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:04,774 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. after waiting 0 ms 2024-11-07T14:20:04,774 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:04,774 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:04,774 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7ebdf008998e34da6a8bfd565c333c77: 2024-11-07T14:20:04,775 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:04,775 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730989204775"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989204775"}]},"ts":"1730989204775"} 2024-11-07T14:20:04,780 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:04,781 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:04,781 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989204781"}]},"ts":"1730989204781"} 2024-11-07T14:20:04,783 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:04,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=7ebdf008998e34da6a8bfd565c333c77, ASSIGN}] 2024-11-07T14:20:04,788 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=7ebdf008998e34da6a8bfd565c333c77, ASSIGN 2024-11-07T14:20:04,790 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=7ebdf008998e34da6a8bfd565c333c77, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:04,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=7ebdf008998e34da6a8bfd565c333c77, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:04,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 7ebdf008998e34da6a8bfd565c333c77, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:05,095 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:05,102 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:05,102 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 7ebdf008998e34da6a8bfd565c333c77, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,105 INFO [StoreOpener-7ebdf008998e34da6a8bfd565c333c77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,108 INFO [StoreOpener-7ebdf008998e34da6a8bfd565c333c77-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ebdf008998e34da6a8bfd565c333c77 columnFamilyName testFamily 2024-11-07T14:20:05,108 DEBUG [StoreOpener-7ebdf008998e34da6a8bfd565c333c77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:05,109 INFO [StoreOpener-7ebdf008998e34da6a8bfd565c333c77-1 {}] regionserver.HStore(327): Store=7ebdf008998e34da6a8bfd565c333c77/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:05,113 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,114 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,117 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:20:05,121 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:05,121 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 7ebdf008998e34da6a8bfd565c333c77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67380622, jitterRate=0.004049509763717651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:05,122 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 7ebdf008998e34da6a8bfd565c333c77: 2024-11-07T14:20:05,124 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., pid=68, masterSystemTime=1730989205095 2024-11-07T14:20:05,126 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:05,126 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:20:05,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=7ebdf008998e34da6a8bfd565c333c77, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:05,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-07T14:20:05,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 7ebdf008998e34da6a8bfd565c333c77, server=69430dbfd73f,36593,1730989059679 in 186 msec 2024-11-07T14:20:05,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T14:20:05,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=7ebdf008998e34da6a8bfd565c333c77, ASSIGN in 344 msec 2024-11-07T14:20:05,134 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:05,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989205134"}]},"ts":"1730989205134"} 2024-11-07T14:20:05,137 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:05,141 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:05,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 804 msec 2024-11-07T14:20:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T14:20:05,346 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-07T14:20:05,346 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:05,346 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:05,351 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:05,351 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:05,351 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T14:20:05,361 DEBUG [hconnection-0x665e473-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:05,364 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47338, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:05,384 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:05,395 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:20:05,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:20:05,901 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668993 expired clientIPAndPort=172.17.0.2:51970, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:20:06,031 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:20:06,091 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-07T14:20:06,092 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-07T14:20:06,093 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-07T14:20:06,825 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:20:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-07T14:20:07,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:20:08,414 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:20:08,415 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T14:20:08.415Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=985ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:20:08,416 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3021 2024-11-07T14:20:08,416 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1cef9240 to 127.0.0.1:49795 2024-11-07T14:20:08,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:08,416 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:08,417 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:08,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:08,417 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f7b8ba2 to 127.0.0.1:49795 2024-11-07T14:20:08,427 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=233 (was 234), OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 374) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5559 (was 5591) 2024-11-07T14:20:08,439 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=233, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=5559 2024-11-07T14:20:08,441 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6afb4947 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c567961 2024-11-07T14:20:08,445 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x6afb4947 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf5aca0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:08,446 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0fe38a51 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60dba738 2024-11-07T14:20:08,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505a6b15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:08,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T14:20:08,457 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:08,457 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:08,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-07T14:20:08,459 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T14:20:08,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741877_1053 (size=424) 2024-11-07T14:20:08,474 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bf49605def8c8556fa53cbe805f31c9f, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:08,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741878_1054 (size=75) 2024-11-07T14:20:08,487 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:08,487 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing bf49605def8c8556fa53cbe805f31c9f, disabling compactions & flushes 2024-11-07T14:20:08,487 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,487 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,487 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. after waiting 0 ms 2024-11-07T14:20:08,487 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,488 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,488 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for bf49605def8c8556fa53cbe805f31c9f: 2024-11-07T14:20:08,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:08,489 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1730989208489"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989208489"}]},"ts":"1730989208489"} 2024-11-07T14:20:08,491 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:08,492 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:08,493 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989208492"}]},"ts":"1730989208492"} 2024-11-07T14:20:08,495 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:08,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bf49605def8c8556fa53cbe805f31c9f, ASSIGN}] 2024-11-07T14:20:08,512 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bf49605def8c8556fa53cbe805f31c9f, ASSIGN 2024-11-07T14:20:08,514 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bf49605def8c8556fa53cbe805f31c9f, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:08,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=bf49605def8c8556fa53cbe805f31c9f, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:08,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure bf49605def8c8556fa53cbe805f31c9f, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:08,820 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:08,830 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,830 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => bf49605def8c8556fa53cbe805f31c9f, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:08,830 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,831 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:08,831 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,831 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,835 INFO [StoreOpener-bf49605def8c8556fa53cbe805f31c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,838 INFO [StoreOpener-bf49605def8c8556fa53cbe805f31c9f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf49605def8c8556fa53cbe805f31c9f columnFamilyName testFamily 2024-11-07T14:20:08,838 DEBUG [StoreOpener-bf49605def8c8556fa53cbe805f31c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:08,839 INFO [StoreOpener-bf49605def8c8556fa53cbe805f31c9f-1 {}] regionserver.HStore(327): Store=bf49605def8c8556fa53cbe805f31c9f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:08,840 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,841 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,845 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:20:08,850 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:08,852 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened bf49605def8c8556fa53cbe805f31c9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62340480, jitterRate=-0.07105445861816406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:08,853 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for bf49605def8c8556fa53cbe805f31c9f: 2024-11-07T14:20:08,854 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f., pid=71, masterSystemTime=1730989208820 2024-11-07T14:20:08,857 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,857 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:20:08,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=bf49605def8c8556fa53cbe805f31c9f, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:08,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T14:20:08,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure bf49605def8c8556fa53cbe805f31c9f, server=69430dbfd73f,36593,1730989059679 in 192 msec 2024-11-07T14:20:08,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-07T14:20:08,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bf49605def8c8556fa53cbe805f31c9f, ASSIGN in 353 msec 2024-11-07T14:20:08,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:08,865 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989208865"}]},"ts":"1730989208865"} 2024-11-07T14:20:08,868 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:08,871 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:08,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 417 msec 2024-11-07T14:20:08,929 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:08,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T14:20:08,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:08,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T14:20:08,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:08,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T14:20:08,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:08,948 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T14:20:08,948 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T14:20:09,461 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-07T14:20:09,461 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:09,461 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:09,466 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:09,466 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:09,466 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T14:20:09,469 DEBUG [hconnection-0x72b46686-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:09,471 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47344, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:09,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:09,482 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:09,482 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:20:09,484 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x6afb4947 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:20:09,484 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x6afb4947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:09,486 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47352, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:20:09,488 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:20:09,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669063 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:09,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:20:09,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669063 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:09,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T14:20:12,095 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-07T14:20:12,495 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3005 2024-11-07T14:20:12,495 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0fe38a51 to 127.0.0.1:49795 2024-11-07T14:20:12,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:12,495 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:12,495 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669063 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:20:12,496 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:12,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:12,496 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6afb4947 to 127.0.0.1:49795 2024-11-07T14:20:12,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628669063' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T14:20:12,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:47352 deadline: 1730989213495, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4907802727628669063' does not exist 2024-11-07T14:20:12,496 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 69430dbfd73f,36593,1730989059679 for closing scanner id = 4907802727628669063 for bf49605def8c8556fa53cbe805f31c9f of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T14:20:12,499 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:12,507 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=232 (was 233), OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=444 (was 448), ProcessCount=11 (was 11), AvailableMemoryMB=5553 (was 5559) 2024-11-07T14:20:12,518 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=232, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=444, ProcessCount=11, AvailableMemoryMB=5552 2024-11-07T14:20:12,520 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55735c9f to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@105539b8 2024-11-07T14:20:12,524 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x55735c9f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705b420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:12,525 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b2f22d9 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13653722 2024-11-07T14:20:12,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3d0675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:12,532 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:12,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-07T14:20:12,534 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:12,534 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:12,535 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-07T14:20:12,535 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T14:20:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741879_1055 (size=416) 2024-11-07T14:20:12,548 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0b2bb17d0bed0d5b693dc62e7f6bbc01, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:12,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741880_1056 (size=67) 2024-11-07T14:20:12,556 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:12,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 0b2bb17d0bed0d5b693dc62e7f6bbc01, disabling compactions & flushes 2024-11-07T14:20:12,557 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. after waiting 0 ms 2024-11-07T14:20:12,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,557 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0b2bb17d0bed0d5b693dc62e7f6bbc01: 2024-11-07T14:20:12,558 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:12,559 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1730989212558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989212558"}]},"ts":"1730989212558"} 2024-11-07T14:20:12,561 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:12,562 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:12,562 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989212562"}]},"ts":"1730989212562"} 2024-11-07T14:20:12,563 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:12,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b2bb17d0bed0d5b693dc62e7f6bbc01, ASSIGN}] 2024-11-07T14:20:12,569 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b2bb17d0bed0d5b693dc62e7f6bbc01, ASSIGN 2024-11-07T14:20:12,570 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b2bb17d0bed0d5b693dc62e7f6bbc01, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:12,720 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0b2bb17d0bed0d5b693dc62e7f6bbc01, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:12,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 0b2bb17d0bed0d5b693dc62e7f6bbc01, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:12,876 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:12,882 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 0b2bb17d0bed0d5b693dc62e7f6bbc01, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:12,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:12,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,885 INFO [StoreOpener-0b2bb17d0bed0d5b693dc62e7f6bbc01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,887 INFO [StoreOpener-0b2bb17d0bed0d5b693dc62e7f6bbc01-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b2bb17d0bed0d5b693dc62e7f6bbc01 columnFamilyName testFamily 2024-11-07T14:20:12,887 DEBUG [StoreOpener-0b2bb17d0bed0d5b693dc62e7f6bbc01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:12,888 INFO [StoreOpener-0b2bb17d0bed0d5b693dc62e7f6bbc01-1 {}] regionserver.HStore(327): Store=0b2bb17d0bed0d5b693dc62e7f6bbc01/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:12,889 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:20:12,898 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:12,898 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 0b2bb17d0bed0d5b693dc62e7f6bbc01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73957365, jitterRate=0.10205061733722687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:12,900 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 0b2bb17d0bed0d5b693dc62e7f6bbc01: 2024-11-07T14:20:12,901 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01., pid=74, masterSystemTime=1730989212876 2024-11-07T14:20:12,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,903 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:20:12,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0b2bb17d0bed0d5b693dc62e7f6bbc01, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:12,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-07T14:20:12,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 0b2bb17d0bed0d5b693dc62e7f6bbc01, server=69430dbfd73f,36593,1730989059679 in 183 msec 2024-11-07T14:20:12,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T14:20:12,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b2bb17d0bed0d5b693dc62e7f6bbc01, ASSIGN in 341 msec 2024-11-07T14:20:12,912 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:12,913 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989212913"}]},"ts":"1730989212913"} 2024-11-07T14:20:12,914 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:12,920 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:12,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 388 msec 2024-11-07T14:20:12,990 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T14:20:13,538 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-07T14:20:13,538 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:13,538 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:13,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-07T14:20:13,545 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669064 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:13,546 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:13,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:13,546 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-07T14:20:13,548 DEBUG [hconnection-0x7f14fddb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:13,549 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43054, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:13,556 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:13,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:13,559 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669067 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:20:13,559 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T14:20:13,611 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628668997 expired clientIPAndPort=172.17.0.2:51978, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:20:14,332 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:20:18,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-07T14:20:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T14:20:18,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:20,060 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-07T14:20:20,060 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b2f22d9 to 127.0.0.1:49795 2024-11-07T14:20:20,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:20,061 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:20,061 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:20,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:20,061 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55735c9f to 127.0.0.1:49795 2024-11-07T14:20:20,072 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231 (was 232), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=409 (was 444), ProcessCount=11 (was 11), AvailableMemoryMB=5489 (was 5552) 2024-11-07T14:20:20,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=409, ProcessCount=11, AvailableMemoryMB=5489 2024-11-07T14:20:20,083 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71ceb8d0 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5fc4b612 2024-11-07T14:20:20,087 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x71ceb8d0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d93041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:20,088 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b0dacce to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cc8b269 2024-11-07T14:20:20,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78449d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:20,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-07T14:20:20,096 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:20,097 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:20,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-07T14:20:20,097 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T14:20:20,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741881_1057 (size=420) 2024-11-07T14:20:20,508 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1db1b29a17977aa78044ceac0b9bb46e, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:20,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741882_1058 (size=71) 2024-11-07T14:20:20,919 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:20,919 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1db1b29a17977aa78044ceac0b9bb46e, disabling compactions & flushes 2024-11-07T14:20:20,919 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:20,919 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:20,919 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. after waiting 0 ms 2024-11-07T14:20:20,919 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:20,919 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:20,920 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1db1b29a17977aa78044ceac0b9bb46e: 2024-11-07T14:20:20,921 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:20,921 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730989220921"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989220921"}]},"ts":"1730989220921"} 2024-11-07T14:20:20,923 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:20,924 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:20,924 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989220924"}]},"ts":"1730989220924"} 2024-11-07T14:20:20,925 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:20,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=1db1b29a17977aa78044ceac0b9bb46e, ASSIGN}] 2024-11-07T14:20:20,931 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=1db1b29a17977aa78044ceac0b9bb46e, ASSIGN 2024-11-07T14:20:20,931 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=1db1b29a17977aa78044ceac0b9bb46e, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:21,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=1db1b29a17977aa78044ceac0b9bb46e, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:21,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 1db1b29a17977aa78044ceac0b9bb46e, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T14:20:21,218 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669003 expired clientIPAndPort=172.17.0.2:54998, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:20:21,235 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:21,240 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:21,240 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => 1db1b29a17977aa78044ceac0b9bb46e, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:21,240 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,240 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:21,240 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for 1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,241 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for 1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,242 INFO [StoreOpener-1db1b29a17977aa78044ceac0b9bb46e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,244 INFO [StoreOpener-1db1b29a17977aa78044ceac0b9bb46e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db1b29a17977aa78044ceac0b9bb46e columnFamilyName testFamily 2024-11-07T14:20:21,244 DEBUG [StoreOpener-1db1b29a17977aa78044ceac0b9bb46e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:21,245 INFO [StoreOpener-1db1b29a17977aa78044ceac0b9bb46e-1 {}] regionserver.HStore(327): Store=1db1b29a17977aa78044ceac0b9bb46e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:21,246 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,246 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,248 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for 1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:20:21,251 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:21,251 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened 1db1b29a17977aa78044ceac0b9bb46e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59951499, jitterRate=-0.10665304958820343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:21,252 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for 1db1b29a17977aa78044ceac0b9bb46e: 2024-11-07T14:20:21,253 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e., pid=77, masterSystemTime=1730989221235 2024-11-07T14:20:21,255 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:21,255 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:20:21,256 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=1db1b29a17977aa78044ceac0b9bb46e, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:21,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T14:20:21,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 1db1b29a17977aa78044ceac0b9bb46e, server=69430dbfd73f,36593,1730989059679 in 173 msec 2024-11-07T14:20:21,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-07T14:20:21,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=1db1b29a17977aa78044ceac0b9bb46e, ASSIGN in 329 msec 2024-11-07T14:20:21,261 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:21,262 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989221261"}]},"ts":"1730989221261"} 2024-11-07T14:20:21,263 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:21,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:21,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1710 sec 2024-11-07T14:20:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T14:20:23,101 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-07T14:20:23,101 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:23,101 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:23,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-07T14:20:23,105 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669068 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:23,107 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:23,107 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:23,107 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-07T14:20:23,109 DEBUG [hconnection-0x25ba0575-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:23,110 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42668, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:23,118 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:23,120 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:23,122 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669071 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T14:20:23,122 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T14:20:24,098 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-07T14:20:24,623 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-07T14:20:24,623 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b0dacce to 127.0.0.1:49795 2024-11-07T14:20:24,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:24,624 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:24,624 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:24,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:24,624 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x71ceb8d0 to 127.0.0.1:49795 2024-11-07T14:20:24,635 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=234 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 409), ProcessCount=11 (was 11), AvailableMemoryMB=6716 (was 5489) - AvailableMemoryMB LEAK? - 2024-11-07T14:20:24,645 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=6716 2024-11-07T14:20:24,646 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66da1ff5 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@725193b4 2024-11-07T14:20:24,661 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x66da1ff5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da5fe14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:24,663 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e09ac14 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8dd9246 2024-11-07T14:20:24,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ece111f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:24,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-07T14:20:24,676 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:24,676 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:24,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-07T14:20:24,677 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T14:20:24,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741883_1059 (size=419) 2024-11-07T14:20:25,097 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => cf9d94195d1c2def97027bb9d0976070, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:25,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741884_1060 (size=70) 2024-11-07T14:20:25,506 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:25,506 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing cf9d94195d1c2def97027bb9d0976070, disabling compactions & flushes 2024-11-07T14:20:25,506 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,506 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,506 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. after waiting 0 ms 2024-11-07T14:20:25,506 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,506 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,506 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for cf9d94195d1c2def97027bb9d0976070: 2024-11-07T14:20:25,507 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:25,508 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1730989225507"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989225507"}]},"ts":"1730989225507"} 2024-11-07T14:20:25,509 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:25,510 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:25,510 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989225510"}]},"ts":"1730989225510"} 2024-11-07T14:20:25,512 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:25,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=cf9d94195d1c2def97027bb9d0976070, ASSIGN}] 2024-11-07T14:20:25,516 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=cf9d94195d1c2def97027bb9d0976070, ASSIGN 2024-11-07T14:20:25,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=cf9d94195d1c2def97027bb9d0976070, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:25,668 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=cf9d94195d1c2def97027bb9d0976070, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:25,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure cf9d94195d1c2def97027bb9d0976070, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T14:20:25,822 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:25,827 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,827 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => cf9d94195d1c2def97027bb9d0976070, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:25,828 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,828 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:25,828 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,828 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,830 INFO [StoreOpener-cf9d94195d1c2def97027bb9d0976070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,832 INFO [StoreOpener-cf9d94195d1c2def97027bb9d0976070-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf9d94195d1c2def97027bb9d0976070 columnFamilyName testFamily 2024-11-07T14:20:25,832 DEBUG [StoreOpener-cf9d94195d1c2def97027bb9d0976070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:25,832 INFO [StoreOpener-cf9d94195d1c2def97027bb9d0976070-1 {}] regionserver.HStore(327): Store=cf9d94195d1c2def97027bb9d0976070/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:25,834 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,834 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,837 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:20:25,840 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:25,840 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened cf9d94195d1c2def97027bb9d0976070; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67675181, jitterRate=0.008438780903816223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:25,842 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for cf9d94195d1c2def97027bb9d0976070: 2024-11-07T14:20:25,843 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070., pid=80, masterSystemTime=1730989225822 2024-11-07T14:20:25,844 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,845 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:20:25,845 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=cf9d94195d1c2def97027bb9d0976070, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:25,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-07T14:20:25,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure cf9d94195d1c2def97027bb9d0976070, server=69430dbfd73f,36593,1730989059679 in 177 msec 2024-11-07T14:20:25,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T14:20:25,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=cf9d94195d1c2def97027bb9d0976070, ASSIGN in 333 msec 2024-11-07T14:20:25,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:25,850 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989225850"}]},"ts":"1730989225850"} 2024-11-07T14:20:25,852 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:25,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:25,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1820 sec 2024-11-07T14:20:26,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:20:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T14:20:27,681 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-07T14:20:27,681 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:27,681 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:27,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-07T14:20:27,686 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669072 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:27,688 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:27,688 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:27,688 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-07T14:20:27,690 DEBUG [hconnection-0x7e148641-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:27,691 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42678, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:27,699 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:27,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:27,703 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:20:27,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669075 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:27,704 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:20:27,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669075 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:27,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T14:20:28,225 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669005 expired clientIPAndPort=172.17.0.2:54998, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:20:28,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T14:20:28,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:28,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-07T14:20:28,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:30,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-07T14:20:30,142 ERROR [LeaseRenewer:jenkins@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:41411,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:20:30,523 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:41411,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:20:30,714 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4007: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4007: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:20:30,715 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3010 2024-11-07T14:20:30,715 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e09ac14 to 127.0.0.1:49795 2024-11-07T14:20:30,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:30,715 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:30,716 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:30,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:30,716 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66da1ff5 to 127.0.0.1:49795 2024-11-07T14:20:30,727 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=229 (was 234), OpenFileDescriptor=451 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=6348 (was 6716) 2024-11-07T14:20:30,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=229, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=6347 2024-11-07T14:20:30,739 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77bd6ab2 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4ad9a500 2024-11-07T14:20:30,744 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x77bd6ab2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341e6768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:30,745 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f774243 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@229b58e3 2024-11-07T14:20:30,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6f4ef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:30,751 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:30,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T14:20:30,753 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:30,753 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:30,753 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-07T14:20:30,754 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T14:20:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741885_1061 (size=421) 2024-11-07T14:20:31,171 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ba1a3fb142dd50d2b08b0509b442a780, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:31,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741886_1062 (size=72) 2024-11-07T14:20:31,209 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:31,586 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:31,586 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing ba1a3fb142dd50d2b08b0509b442a780, disabling compactions & flushes 2024-11-07T14:20:31,586 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,586 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,586 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. after waiting 0 ms 2024-11-07T14:20:31,586 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,586 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,586 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for ba1a3fb142dd50d2b08b0509b442a780: 2024-11-07T14:20:31,587 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:31,588 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730989231588"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989231588"}]},"ts":"1730989231588"} 2024-11-07T14:20:31,589 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:31,590 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:31,590 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989231590"}]},"ts":"1730989231590"} 2024-11-07T14:20:31,592 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:31,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=ba1a3fb142dd50d2b08b0509b442a780, ASSIGN}] 2024-11-07T14:20:31,597 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=ba1a3fb142dd50d2b08b0509b442a780, ASSIGN 2024-11-07T14:20:31,599 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=ba1a3fb142dd50d2b08b0509b442a780, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:31,750 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=ba1a3fb142dd50d2b08b0509b442a780, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:31,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure ba1a3fb142dd50d2b08b0509b442a780, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T14:20:31,903 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:31,909 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => ba1a3fb142dd50d2b08b0509b442a780, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:31,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:31,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,911 INFO [StoreOpener-ba1a3fb142dd50d2b08b0509b442a780-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,913 INFO [StoreOpener-ba1a3fb142dd50d2b08b0509b442a780-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba1a3fb142dd50d2b08b0509b442a780 columnFamilyName testFamily 2024-11-07T14:20:31,913 DEBUG [StoreOpener-ba1a3fb142dd50d2b08b0509b442a780-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:31,914 INFO [StoreOpener-ba1a3fb142dd50d2b08b0509b442a780-1 {}] regionserver.HStore(327): Store=ba1a3fb142dd50d2b08b0509b442a780/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:31,914 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,915 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,917 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:20:31,919 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:31,920 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened ba1a3fb142dd50d2b08b0509b442a780; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64428966, jitterRate=-0.039933592081069946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:31,921 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for ba1a3fb142dd50d2b08b0509b442a780: 2024-11-07T14:20:31,921 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780., pid=83, masterSystemTime=1730989231903 2024-11-07T14:20:31,923 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,923 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:20:31,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=ba1a3fb142dd50d2b08b0509b442a780, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:31,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T14:20:31,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure ba1a3fb142dd50d2b08b0509b442a780, server=69430dbfd73f,36593,1730989059679 in 173 msec 2024-11-07T14:20:31,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-07T14:20:31,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=ba1a3fb142dd50d2b08b0509b442a780, ASSIGN in 330 msec 2024-11-07T14:20:31,928 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:31,928 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989231928"}]},"ts":"1730989231928"} 2024-11-07T14:20:31,929 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:31,932 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:31,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1820 sec 2024-11-07T14:20:33,229 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669010 expired clientIPAndPort=172.17.0.2:33606, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:20:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T14:20:33,759 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-07T14:20:33,759 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:33,759 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:33,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-07T14:20:33,770 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669076 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:33,771 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:33,771 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:33,771 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T14:20:33,782 DEBUG [hconnection-0x2155c27c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:33,784 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49656, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:33,793 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:33,796 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:33,803 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:20:33,803 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T14:20:35,331 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669015 expired clientIPAndPort=172.17.0.2:59508, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:20:36,101 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-07T14:20:36,825 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:20:37,433 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669017 expired clientIPAndPort=172.17.0.2:59508, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:20:37,453 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:20:38,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T14:20:38,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-07T14:20:40,810 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:20:40,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:49656 deadline: 1730989242802, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T14:20:40,811 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T14:20:40,811 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4907802727628669079 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T14:20:40,812 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7016 2024-11-07T14:20:40,812 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f774243 to 127.0.0.1:49795 2024-11-07T14:20:40,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:40,812 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:40,812 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:40,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:40,813 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x77bd6ab2 to 127.0.0.1:49795 2024-11-07T14:20:40,822 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 229) - Thread LEAK? -, OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=293 (was 347), ProcessCount=11 (was 11), AvailableMemoryMB=6143 (was 6347) 2024-11-07T14:20:40,830 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=6143 2024-11-07T14:20:40,832 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x04c8174b to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68f5913 2024-11-07T14:20:40,835 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x04c8174b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777f2809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:40,836 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74815982 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f99cb87 2024-11-07T14:20:40,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2fed70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:40,841 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:40,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T14:20:40,843 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:40,843 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:40,843 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-07T14:20:40,844 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T14:20:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741887_1063 (size=427) 2024-11-07T14:20:41,253 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 17f066662d05461282843a956500594e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:41,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741888_1064 (size=78) 2024-11-07T14:20:41,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:41,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 17f066662d05461282843a956500594e, disabling compactions & flushes 2024-11-07T14:20:41,663 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. after waiting 0 ms 2024-11-07T14:20:41,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,663 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 17f066662d05461282843a956500594e: 2024-11-07T14:20:41,665 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:41,665 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1730989241665"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989241665"}]},"ts":"1730989241665"} 2024-11-07T14:20:41,667 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:41,667 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:41,667 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989241667"}]},"ts":"1730989241667"} 2024-11-07T14:20:41,669 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:41,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=17f066662d05461282843a956500594e, ASSIGN}] 2024-11-07T14:20:41,673 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=17f066662d05461282843a956500594e, ASSIGN 2024-11-07T14:20:41,673 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=17f066662d05461282843a956500594e, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:41,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=17f066662d05461282843a956500594e, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:41,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure 17f066662d05461282843a956500594e, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T14:20:41,977 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:41,984 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => 17f066662d05461282843a956500594e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:41,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true 17f066662d05461282843a956500594e 2024-11-07T14:20:41,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:41,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for 17f066662d05461282843a956500594e 2024-11-07T14:20:41,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for 17f066662d05461282843a956500594e 2024-11-07T14:20:41,986 INFO [StoreOpener-17f066662d05461282843a956500594e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 17f066662d05461282843a956500594e 2024-11-07T14:20:41,988 INFO [StoreOpener-17f066662d05461282843a956500594e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17f066662d05461282843a956500594e columnFamilyName testFamily 2024-11-07T14:20:41,988 DEBUG [StoreOpener-17f066662d05461282843a956500594e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:41,988 INFO [StoreOpener-17f066662d05461282843a956500594e-1 {}] regionserver.HStore(327): Store=17f066662d05461282843a956500594e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:41,989 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e 2024-11-07T14:20:41,989 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e 2024-11-07T14:20:41,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for 17f066662d05461282843a956500594e 2024-11-07T14:20:41,994 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:41,995 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened 17f066662d05461282843a956500594e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67902123, jitterRate=0.011820480227470398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:41,996 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for 17f066662d05461282843a956500594e: 2024-11-07T14:20:41,996 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., pid=86, masterSystemTime=1730989241977 2024-11-07T14:20:41,998 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,998 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:20:41,999 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=17f066662d05461282843a956500594e, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:42,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-07T14:20:42,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure 17f066662d05461282843a956500594e, server=69430dbfd73f,36593,1730989059679 in 174 msec 2024-11-07T14:20:42,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T14:20:42,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=17f066662d05461282843a956500594e, ASSIGN in 330 msec 2024-11-07T14:20:42,004 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:42,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989242004"}]},"ts":"1730989242004"} 2024-11-07T14:20:42,005 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:42,007 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:42,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1660 sec 2024-11-07T14:20:42,102 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T14:20:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T14:20:43,847 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-07T14:20:43,847 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:43,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:43,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-07T14:20:43,851 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669081 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:43,852 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:43,852 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:43,853 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T14:20:43,854 DEBUG [hconnection-0x28c562a0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:43,856 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39506, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:43,863 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:43,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:20:43,865 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:20:43,866 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x04c8174b {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:20:43,866 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x04c8174b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:43,867 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39508, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:20:43,869 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:20:45,541 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669031 expired clientIPAndPort=172.17.0.2:59536, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:20:47,486 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:20:48,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T14:20:48,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:49,875 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2024-11-07T14:20:49,875 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2024-11-07T14:20:49,875 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:20:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-07T14:20:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:20:50,892 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T14:20:52,550 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669033 expired clientIPAndPort=172.17.0.2:59536, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:20:52,874 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1973ms 2024-11-07T14:20:52,874 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1973ms 2024-11-07T14:20:52,874 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:20:52,875 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T14:20:49.875Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2024-11-07T14:20:52.874Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1973ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T14:20:52,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9011 2024-11-07T14:20:52,876 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74815982 to 127.0.0.1:49795 2024-11-07T14:20:52,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:52,876 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:20:52,876 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:20:52,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:20:52,876 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x04c8174b to 127.0.0.1:49795 2024-11-07T14:20:52,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=226 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=270 (was 293), ProcessCount=11 (was 11), AvailableMemoryMB=6088 (was 6143) 2024-11-07T14:20:52,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=270, ProcessCount=11, AvailableMemoryMB=6088 2024-11-07T14:20:52,895 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66e7829f to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f034653 2024-11-07T14:20:52,900 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x66e7829f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2fc934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:52,901 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0892b942 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5ae4714c 2024-11-07T14:20:52,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de3f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:20:52,906 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:20:52,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-07T14:20:52,908 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:20:52,908 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:52,908 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-07T14:20:52,909 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:20:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T14:20:52,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741889_1065 (size=415) 2024-11-07T14:20:53,320 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => aeb7f5460c70a1472f020d629c072193, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:20:53,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741890_1066 (size=66) 2024-11-07T14:20:53,728 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:53,728 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing aeb7f5460c70a1472f020d629c072193, disabling compactions & flushes 2024-11-07T14:20:53,728 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:53,728 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:53,728 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. after waiting 0 ms 2024-11-07T14:20:53,728 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:53,728 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:53,728 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for aeb7f5460c70a1472f020d629c072193: 2024-11-07T14:20:53,729 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:20:53,729 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1730989253729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989253729"}]},"ts":"1730989253729"} 2024-11-07T14:20:53,731 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:20:53,731 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:20:53,732 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989253731"}]},"ts":"1730989253731"} 2024-11-07T14:20:53,733 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:20:53,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=aeb7f5460c70a1472f020d629c072193, ASSIGN}] 2024-11-07T14:20:53,737 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=aeb7f5460c70a1472f020d629c072193, ASSIGN 2024-11-07T14:20:53,738 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=aeb7f5460c70a1472f020d629c072193, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:20:53,888 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=aeb7f5460c70a1472f020d629c072193, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:53,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure aeb7f5460c70a1472f020d629c072193, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:20:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T14:20:54,042 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:20:54,048 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:54,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => aeb7f5460c70a1472f020d629c072193, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:20:54,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:20:54,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,051 INFO [StoreOpener-aeb7f5460c70a1472f020d629c072193-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,052 INFO [StoreOpener-aeb7f5460c70a1472f020d629c072193-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeb7f5460c70a1472f020d629c072193 columnFamilyName testFamily 2024-11-07T14:20:54,052 DEBUG [StoreOpener-aeb7f5460c70a1472f020d629c072193-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:20:54,053 INFO [StoreOpener-aeb7f5460c70a1472f020d629c072193-1 {}] regionserver.HStore(327): Store=aeb7f5460c70a1472f020d629c072193/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:20:54,053 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,054 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,056 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for aeb7f5460c70a1472f020d629c072193 2024-11-07T14:20:54,058 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:20:54,058 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened aeb7f5460c70a1472f020d629c072193; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62123663, jitterRate=-0.07428528368473053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:20:54,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for aeb7f5460c70a1472f020d629c072193: 2024-11-07T14:20:54,060 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193., pid=89, masterSystemTime=1730989254042 2024-11-07T14:20:54,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:54,061 INFO [RS_OPEN_PRIORITY_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:20:54,062 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=aeb7f5460c70a1472f020d629c072193, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:20:54,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T14:20:54,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure aeb7f5460c70a1472f020d629c072193, server=69430dbfd73f,36593,1730989059679 in 173 msec 2024-11-07T14:20:54,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T14:20:54,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=aeb7f5460c70a1472f020d629c072193, ASSIGN in 329 msec 2024-11-07T14:20:54,066 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:20:54,067 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989254066"}]},"ts":"1730989254066"} 2024-11-07T14:20:54,068 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:20:54,070 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:20:54,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1640 sec 2024-11-07T14:20:54,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-07T14:20:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T14:20:55,912 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-07T14:20:55,912 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:20:55,912 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:55,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-07T14:20:55,916 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669088 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:55,917 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T14:20:55,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:20:55,917 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-07T14:20:55,919 DEBUG [hconnection-0x43a16bd3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:20:55,920 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54756, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:20:55,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:20:55,928 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T14:20:55,929 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T14:20:55,930 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669091 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:55,931 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T14:20:55,931 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669091 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:20:55,932 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T14:20:57,392 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:20:58,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T14:20:58,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:20:59,584 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:21:00,657 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669038 expired clientIPAndPort=172.17.0.2:49996, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:21:00,764 ERROR [LeaseRenewer:jenkins@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:41411,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:21:01,258 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669042 expired clientIPAndPort=172.17.0.2:47306, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:01,258 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669044 expired clientIPAndPort=172.17.0.2:47306, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:01,258 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669045 expired clientIPAndPort=172.17.0.2:47306, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:01,921 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:41411 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:41411,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:21:04,361 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669051 expired clientIPAndPort=172.17.0.2:47322, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:21:04,935 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10004: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10004: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T14:21:04,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9004 2024-11-07T14:21:04,936 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0892b942 to 127.0.0.1:49795 2024-11-07T14:21:04,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:04,936 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:21:04,936 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:21:04,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:04,936 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66e7829f to 127.0.0.1:49795 2024-11-07T14:21:04,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226 (was 226), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=236 (was 270), ProcessCount=11 (was 11), AvailableMemoryMB=5955 (was 6088) 2024-11-07T14:21:04,953 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=11, AvailableMemoryMB=5955 2024-11-07T14:21:04,954 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x609296c6 to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c2ac92b 2024-11-07T14:21:04,957 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x609296c6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bbbb9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:21:04,958 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e4092ac to 127.0.0.1:49795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3f296590 2024-11-07T14:21:04,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf9e015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:21:04,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:21:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T14:21:04,965 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:21:04,965 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:21:04,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-07T14:21:04,965 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:21:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T14:21:04,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741891_1067 (size=431) 2024-11-07T14:21:05,375 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 720e6614d7d2c100ffc7df45089cb601, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452 2024-11-07T14:21:05,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741892_1068 (size=82) 2024-11-07T14:21:05,432 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:21:05,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:21:05,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 720e6614d7d2c100ffc7df45089cb601, disabling compactions & flushes 2024-11-07T14:21:05,782 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:05,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:05,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. after waiting 0 ms 2024-11-07T14:21:05,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:05,782 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:05,783 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 720e6614d7d2c100ffc7df45089cb601: 2024-11-07T14:21:05,784 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:21:05,784 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1730989265784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730989265784"}]},"ts":"1730989265784"} 2024-11-07T14:21:05,785 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T14:21:05,786 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:21:05,786 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989265786"}]},"ts":"1730989265786"} 2024-11-07T14:21:05,787 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T14:21:05,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=720e6614d7d2c100ffc7df45089cb601, ASSIGN}] 2024-11-07T14:21:05,792 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=720e6614d7d2c100ffc7df45089cb601, ASSIGN 2024-11-07T14:21:05,792 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=720e6614d7d2c100ffc7df45089cb601, ASSIGN; state=OFFLINE, location=69430dbfd73f,36593,1730989059679; forceNewPlan=false, retain=false 2024-11-07T14:21:05,943 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=720e6614d7d2c100ffc7df45089cb601, regionState=OPENING, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:21:05,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 720e6614d7d2c100ffc7df45089cb601, server=69430dbfd73f,36593,1730989059679}] 2024-11-07T14:21:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T14:21:06,097 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 69430dbfd73f,36593,1730989059679 2024-11-07T14:21:06,102 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:06,102 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 720e6614d7d2c100ffc7df45089cb601, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:21:06,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:21:06,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,103 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,105 INFO [StoreOpener-720e6614d7d2c100ffc7df45089cb601-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,106 INFO [StoreOpener-720e6614d7d2c100ffc7df45089cb601-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 720e6614d7d2c100ffc7df45089cb601 columnFamilyName testFamily 2024-11-07T14:21:06,107 DEBUG [StoreOpener-720e6614d7d2c100ffc7df45089cb601-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:21:06,107 INFO [StoreOpener-720e6614d7d2c100ffc7df45089cb601-1 {}] regionserver.HStore(327): Store=720e6614d7d2c100ffc7df45089cb601/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:21:06,108 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,108 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,111 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:06,113 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:21:06,113 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 720e6614d7d2c100ffc7df45089cb601; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74141868, jitterRate=0.10479992628097534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T14:21:06,114 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 720e6614d7d2c100ffc7df45089cb601: 2024-11-07T14:21:06,115 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., pid=92, masterSystemTime=1730989266097 2024-11-07T14:21:06,116 DEBUG [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:06,116 INFO [RS_OPEN_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:06,117 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=720e6614d7d2c100ffc7df45089cb601, regionState=OPEN, openSeqNum=2, regionLocation=69430dbfd73f,36593,1730989059679 2024-11-07T14:21:06,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T14:21:06,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 720e6614d7d2c100ffc7df45089cb601, server=69430dbfd73f,36593,1730989059679 in 173 msec 2024-11-07T14:21:06,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T14:21:06,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=720e6614d7d2c100ffc7df45089cb601, ASSIGN in 329 msec 2024-11-07T14:21:06,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:21:06,122 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989266122"}]},"ts":"1730989266122"} 2024-11-07T14:21:06,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T14:21:06,127 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:21:06,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1640 sec 2024-11-07T14:21:06,825 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:21:06,963 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669056 expired clientIPAndPort=172.17.0.2:47338, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42745 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T14:21:07,968 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-07T14:21:07,969 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T14:21:07,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:21:07,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36593 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-07T14:21:07,972 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4907802727628669092 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T14:21:07,973 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T14:21:07,973 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T14:21:07,974 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T14:21:07,975 DEBUG [hconnection-0x5a3c7d2d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:21:07,977 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34716, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:21:07,983 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T14:21:07,985 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T14:21:07,985 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T14:21:07,986 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x609296c6 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=69430dbfd73f,36593,1730989059679, seqNum=-1] 2024-11-07T14:21:07,987 DEBUG [ReadOnlyZKClient-127.0.0.1:49795@0x609296c6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:21:07,988 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34732, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:21:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:21:07,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:21:08,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T14:21:08,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T14:21:08,965 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669058 expired clientIPAndPort=172.17.0.2:47338, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:08,994 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-07T14:21:08,994 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-07T14:21:08,994 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:21:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-07T14:21:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2] 2024-11-07T14:21:10,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36593 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T14:21:10,994 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=983ms, rpcTimeout=974ms 2024-11-07T14:21:10,994 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=983ms, rpcTimeout=974ms 2024-11-07T14:21:10,994 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601., hostname=69430dbfd73f,36593,1730989059679, seqNum=2 from cache 2024-11-07T14:21:10,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T14:21:08.994Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-07T14:21:10.994Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=983ms, rpcTimeout=974ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=69430dbfd73f:36593 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=983ms, rpcTimeout=974ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=983ms, rpcTimeout=974ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T14:21:10,995 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3010 2024-11-07T14:21:10,996 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e4092ac to 127.0.0.1:49795 2024-11-07T14:21:10,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:10,996 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T14:21:10,996 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:21:10,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:10,996 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x609296c6 to 127.0.0.1:49795 2024-11-07T14:21:11,006 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=232 (was 226) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 236), ProcessCount=11 (was 11), AvailableMemoryMB=5852 (was 5955) 2024-11-07T14:21:11,007 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-07T14:21:11,007 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-07T14:21:11,007 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0cd335b7 to 127.0.0.1:49795 2024-11-07T14:21:11,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:11,007 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T14:21:11,007 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1610520369, stopped=false 2024-11-07T14:21:11,007 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=69430dbfd73f,42745,1730989058075 2024-11-07T14:21:11,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T14:21:11,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:21:11,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-07T14:21:11,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T14:21:11,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:21:11,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:11,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:21:11,010 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '69430dbfd73f,36593,1730989059679' ***** 2024-11-07T14:21:11,010 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-07T14:21:11,011 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T14:21:11,011 INFO [RS:0;69430dbfd73f:36593 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T14:21:11,011 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-07T14:21:11,011 INFO [RS:0;69430dbfd73f:36593 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T14:21:11,011 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for cf9d94195d1c2def97027bb9d0976070 2024-11-07T14:21:11,011 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for f18b2a090aac227c596c7ae1e30557ce 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 9ec939c454be0bf86a25c1d58e445783 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 720e6614d7d2c100ffc7df45089cb601 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for a9b718113897f63098bd30ff08ddfe3c 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for d32a9e0dc85dc68730667345f4bf823f 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 89b3721db112bdfe9245cf965df47224 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 9706565c662723050a81bf87f4213369 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 6fbbea4d395a3aec7237b0e5e1ceaae5 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for bf49605def8c8556fa53cbe805f31c9f 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 0b2bb17d0bed0d5b693dc62e7f6bbc01 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 4f1f1bdcd9e70c414da10881c0097694 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 93cfb54d75f976bc77e2ff0d4c799e55 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for aeb7f5460c70a1472f020d629c072193 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 1db1b29a17977aa78044ceac0b9bb46e 2024-11-07T14:21:11,012 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing cf9d94195d1c2def97027bb9d0976070, disabling compactions & flushes 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 0e797cba1bb85ec7f7b6bb605c8e5361 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 17f066662d05461282843a956500594e 2024-11-07T14:21:11,012 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:11,012 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for b6f9ffe8865cf5861efa2412fd098b17 2024-11-07T14:21:11,012 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. after waiting 0 ms 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for d04a9d74f6b94963a147e616bf79af65 2024-11-07T14:21:11,012 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for ba1a3fb142dd50d2b08b0509b442a780 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 7ebdf008998e34da6a8bfd565c333c77 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 8e7403a5d1cd160a13036a66684da366 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 6c03090e181c03a5de5603313432cd9a 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for b93ccd845b07475bea616303fbd6a1a6 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 6db8379cd8b72d893208d9bc43753e66 2024-11-07T14:21:11,012 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:11,013 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1224): stopping server 69430dbfd73f,36593,1730989059679 2024-11-07T14:21:11,013 DEBUG [RS:0;69430dbfd73f:36593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:11,013 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T14:21:11,013 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T14:21:11,013 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T14:21:11,013 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing cf9d94195d1c2def97027bb9d0976070 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:11,013 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-07T14:21:11,013 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:21:11,013 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-07T14:21:11,013 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T14:21:11,013 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T14:21:11,013 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T14:21:11,013 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T14:21:11,013 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T14:21:11,014 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-07T14:21:11,014 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 1588230740, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, cf9d94195d1c2def97027bb9d0976070, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f18b2a090aac227c596c7ae1e30557ce, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:11,069 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/.tmp/testFamily/27db7a5391e043558b6781df65c71391 is 61, key is row-0/testFamily:testQualifier/1730989227693/Put/seqid=0 2024-11-07T14:21:11,076 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/.tmp/info/7b2f05079a724222acd67c7a1d1ba10c is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3./info:regioninfo/1730989093560/Put/seqid=0 2024-11-07T14:21:11,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741893_1069 (size=5049) 2024-11-07T14:21:11,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741894_1070 (size=51873) 2024-11-07T14:21:11,082 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/.tmp/info/7b2f05079a724222acd67c7a1d1ba10c 2024-11-07T14:21:11,149 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/.tmp/table/eb4dfddf5f5348519a922598d5edc977 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1730989093577/Put/seqid=0 2024-11-07T14:21:11,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741895_1071 (size=9821) 2024-11-07T14:21:11,214 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 1588230740, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, cf9d94195d1c2def97027bb9d0976070, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f18b2a090aac227c596c7ae1e30557ce, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:11,415 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 1588230740, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, cf9d94195d1c2def97027bb9d0976070, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f18b2a090aac227c596c7ae1e30557ce, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:11,481 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/.tmp/testFamily/27db7a5391e043558b6781df65c71391 2024-11-07T14:21:11,489 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/.tmp/testFamily/27db7a5391e043558b6781df65c71391 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/testFamily/27db7a5391e043558b6781df65c71391 2024-11-07T14:21:11,495 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/testFamily/27db7a5391e043558b6781df65c71391, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:11,500 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cf9d94195d1c2def97027bb9d0976070 in 487ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:11,505 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_1_-true/cf9d94195d1c2def97027bb9d0976070/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:11,507 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:21:11,507 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for cf9d94195d1c2def97027bb9d0976070: 2024-11-07T14:21:11,508 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1730989224674.cf9d94195d1c2def97027bb9d0976070. 2024-11-07T14:21:11,508 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f18b2a090aac227c596c7ae1e30557ce, disabling compactions & flushes 2024-11-07T14:21:11,508 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:21:11,508 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:21:11,508 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. after waiting 0 ms 2024-11-07T14:21:11,508 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:21:11,508 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f18b2a090aac227c596c7ae1e30557ce 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:11,512 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T14:21:11,524 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/.tmp/testFamily/1e01b713f9c24fc496a4946e5661ade6 is 61, key is row-0/testFamily:testQualifier/1730989107497/Put/seqid=0 2024-11-07T14:21:11,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741896_1072 (size=5049) 2024-11-07T14:21:11,555 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/.tmp/table/eb4dfddf5f5348519a922598d5edc977 2024-11-07T14:21:11,563 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/.tmp/info/7b2f05079a724222acd67c7a1d1ba10c as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/info/7b2f05079a724222acd67c7a1d1ba10c 2024-11-07T14:21:11,570 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/info/7b2f05079a724222acd67c7a1d1ba10c, entries=290, sequenceid=149, filesize=50.7 K 2024-11-07T14:21:11,571 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/.tmp/table/eb4dfddf5f5348519a922598d5edc977 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/table/eb4dfddf5f5348519a922598d5edc977 2024-11-07T14:21:11,578 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/table/eb4dfddf5f5348519a922598d5edc977, entries=58, sequenceid=149, filesize=9.6 K 2024-11-07T14:21:11,579 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 566ms, sequenceid=149, compaction requested=false 2024-11-07T14:21:11,583 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-07T14:21:11,584 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T14:21:11,584 INFO [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T14:21:11,584 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T14:21:11,585 DEBUG [RS_CLOSE_META-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T14:21:11,615 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f18b2a090aac227c596c7ae1e30557ce, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:11,639 INFO [regionserver/69430dbfd73f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T14:21:11,639 INFO [regionserver/69430dbfd73f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T14:21:11,642 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:21:11,815 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f18b2a090aac227c596c7ae1e30557ce, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:11,929 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/.tmp/testFamily/1e01b713f9c24fc496a4946e5661ade6 2024-11-07T14:21:11,937 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/.tmp/testFamily/1e01b713f9c24fc496a4946e5661ade6 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/testFamily/1e01b713f9c24fc496a4946e5661ade6 2024-11-07T14:21:11,943 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/testFamily/1e01b713f9c24fc496a4946e5661ade6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:11,944 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f18b2a090aac227c596c7ae1e30557ce in 436ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:11,948 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f18b2a090aac227c596c7ae1e30557ce/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:11,949 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:21:11,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f18b2a090aac227c596c7ae1e30557ce: 2024-11-07T14:21:11,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730989106469.f18b2a090aac227c596c7ae1e30557ce. 2024-11-07T14:21:11,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9ec939c454be0bf86a25c1d58e445783, disabling compactions & flushes 2024-11-07T14:21:11,949 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:21:11,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:21:11,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. after waiting 0 ms 2024-11-07T14:21:11,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:21:11,949 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9ec939c454be0bf86a25c1d58e445783 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:11,965 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/.tmp/testFamily/d7175ac7a93f43f1b33836b89edd34c3 is 61, key is row-0/testFamily:testQualifier/1730989202256/Put/seqid=0 2024-11-07T14:21:11,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741897_1073 (size=5049) 2024-11-07T14:21:12,015 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-07T14:21:12,016 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:12,106 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T14:21:12,216 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, 9ec939c454be0bf86a25c1d58e445783, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:12,370 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/.tmp/testFamily/d7175ac7a93f43f1b33836b89edd34c3 2024-11-07T14:21:12,379 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/.tmp/testFamily/d7175ac7a93f43f1b33836b89edd34c3 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/testFamily/d7175ac7a93f43f1b33836b89edd34c3 2024-11-07T14:21:12,386 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/testFamily/d7175ac7a93f43f1b33836b89edd34c3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:12,387 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9ec939c454be0bf86a25c1d58e445783 in 438ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:12,391 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_1_-true/9ec939c454be0bf86a25c1d58e445783/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:12,392 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:21:12,392 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9ec939c454be0bf86a25c1d58e445783: 2024-11-07T14:21:12,392 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1730989201216.9ec939c454be0bf86a25c1d58e445783. 2024-11-07T14:21:12,392 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 720e6614d7d2c100ffc7df45089cb601, disabling compactions & flushes 2024-11-07T14:21:12,393 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:12,393 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:12,393 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. after waiting 0 ms 2024-11-07T14:21:12,393 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:12,393 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 720e6614d7d2c100ffc7df45089cb601 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:12,408 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/.tmp/testFamily/57738cd6cede412d917e694a6f2bdd34 is 61, key is row-0/testFamily:testQualifier/1730989267978/Put/seqid=0 2024-11-07T14:21:12,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741898_1074 (size=5049) 2024-11-07T14:21:12,416 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:12,616 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:12,814 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/.tmp/testFamily/57738cd6cede412d917e694a6f2bdd34 2024-11-07T14:21:12,817 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 720e6614d7d2c100ffc7df45089cb601, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:12,822 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/.tmp/testFamily/57738cd6cede412d917e694a6f2bdd34 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/testFamily/57738cd6cede412d917e694a6f2bdd34 2024-11-07T14:21:12,827 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/testFamily/57738cd6cede412d917e694a6f2bdd34, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:12,828 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 720e6614d7d2c100ffc7df45089cb601 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:12,832 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/720e6614d7d2c100ffc7df45089cb601/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:12,833 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:12,833 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 720e6614d7d2c100ffc7df45089cb601: 2024-11-07T14:21:12,833 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730989264962.720e6614d7d2c100ffc7df45089cb601. 2024-11-07T14:21:12,833 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a9b718113897f63098bd30ff08ddfe3c, disabling compactions & flushes 2024-11-07T14:21:12,833 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:21:12,833 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:21:12,833 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. after waiting 0 ms 2024-11-07T14:21:12,833 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:21:12,833 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a9b718113897f63098bd30ff08ddfe3c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:12,848 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/.tmp/testFamily/d7c7295dfb9448c29e4197822a811dc8 is 61, key is row-0/testFamily:testQualifier/1730989146518/Put/seqid=0 2024-11-07T14:21:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741899_1075 (size=5049) 2024-11-07T14:21:13,017 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-07T14:21:13,017 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:13,068 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 4907802727628669063 expired clientIPAndPort=172.17.0.2:47352, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:21:13,217 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, a9b718113897f63098bd30ff08ddfe3c, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:13,253 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/.tmp/testFamily/d7c7295dfb9448c29e4197822a811dc8 2024-11-07T14:21:13,261 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/.tmp/testFamily/d7c7295dfb9448c29e4197822a811dc8 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/testFamily/d7c7295dfb9448c29e4197822a811dc8 2024-11-07T14:21:13,266 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/testFamily/d7c7295dfb9448c29e4197822a811dc8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:13,267 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a9b718113897f63098bd30ff08ddfe3c in 434ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:13,271 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/a9b718113897f63098bd30ff08ddfe3c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:13,272 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:21:13,272 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a9b718113897f63098bd30ff08ddfe3c: 2024-11-07T14:21:13,272 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730989145470.a9b718113897f63098bd30ff08ddfe3c. 2024-11-07T14:21:13,272 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d32a9e0dc85dc68730667345f4bf823f, disabling compactions & flushes 2024-11-07T14:21:13,272 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:21:13,272 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:21:13,272 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. after waiting 0 ms 2024-11-07T14:21:13,272 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:21:13,272 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d32a9e0dc85dc68730667345f4bf823f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:13,288 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/.tmp/testFamily/7bdcc93c98e948e8af423bef059c7e2b is 61, key is row-0/testFamily:testQualifier/1730989173810/Put/seqid=0 2024-11-07T14:21:13,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741900_1076 (size=5049) 2024-11-07T14:21:13,418 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:13,618 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, d32a9e0dc85dc68730667345f4bf823f, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:13,693 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/.tmp/testFamily/7bdcc93c98e948e8af423bef059c7e2b 2024-11-07T14:21:13,700 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/.tmp/testFamily/7bdcc93c98e948e8af423bef059c7e2b as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/testFamily/7bdcc93c98e948e8af423bef059c7e2b 2024-11-07T14:21:13,706 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/testFamily/7bdcc93c98e948e8af423bef059c7e2b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:13,707 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d32a9e0dc85dc68730667345f4bf823f in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:13,710 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/d32a9e0dc85dc68730667345f4bf823f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:13,711 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:21:13,711 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d32a9e0dc85dc68730667345f4bf823f: 2024-11-07T14:21:13,711 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730989172790.d32a9e0dc85dc68730667345f4bf823f. 2024-11-07T14:21:13,711 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 89b3721db112bdfe9245cf965df47224, disabling compactions & flushes 2024-11-07T14:21:13,711 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:21:13,711 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:21:13,711 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. after waiting 0 ms 2024-11-07T14:21:13,711 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:21:13,711 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 89b3721db112bdfe9245cf965df47224 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:13,728 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/.tmp/testFamily/494d68460c9a44ecac486f6ef63e4048 is 61, key is row-0/testFamily:testQualifier/1730989097379/Put/seqid=0 2024-11-07T14:21:13,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741901_1077 (size=5049) 2024-11-07T14:21:13,818 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:14,018 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-07T14:21:14,019 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 89b3721db112bdfe9245cf965df47224, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:14,133 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/.tmp/testFamily/494d68460c9a44ecac486f6ef63e4048 2024-11-07T14:21:14,140 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/.tmp/testFamily/494d68460c9a44ecac486f6ef63e4048 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/testFamily/494d68460c9a44ecac486f6ef63e4048 2024-11-07T14:21:14,148 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/testFamily/494d68460c9a44ecac486f6ef63e4048, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:14,148 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 89b3721db112bdfe9245cf965df47224 in 437ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:14,153 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/89b3721db112bdfe9245cf965df47224/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:14,153 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:21:14,153 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 89b3721db112bdfe9245cf965df47224: 2024-11-07T14:21:14,154 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730989094347.89b3721db112bdfe9245cf965df47224. 2024-11-07T14:21:14,154 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9706565c662723050a81bf87f4213369, disabling compactions & flushes 2024-11-07T14:21:14,154 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:21:14,154 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:21:14,154 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. after waiting 0 ms 2024-11-07T14:21:14,154 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:21:14,154 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9706565c662723050a81bf87f4213369 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:14,169 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/.tmp/testFamily/df23130c71bc4b359b06f7bc24f92723 is 61, key is row-0/testFamily:testQualifier/1730989144287/Put/seqid=0 2024-11-07T14:21:14,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741902_1078 (size=5049) 2024-11-07T14:21:14,219 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:14,419 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, 9706565c662723050a81bf87f4213369, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:14,575 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/.tmp/testFamily/df23130c71bc4b359b06f7bc24f92723 2024-11-07T14:21:14,582 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/.tmp/testFamily/df23130c71bc4b359b06f7bc24f92723 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/testFamily/df23130c71bc4b359b06f7bc24f92723 2024-11-07T14:21:14,588 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/testFamily/df23130c71bc4b359b06f7bc24f92723, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:14,589 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9706565c662723050a81bf87f4213369 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:14,593 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNext_0_-false/9706565c662723050a81bf87f4213369/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:14,594 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:21:14,594 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9706565c662723050a81bf87f4213369: 2024-11-07T14:21:14,594 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1730989143254.9706565c662723050a81bf87f4213369. 2024-11-07T14:21:14,594 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6fbbea4d395a3aec7237b0e5e1ceaae5, disabling compactions & flushes 2024-11-07T14:21:14,594 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:14,594 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:14,594 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. after waiting 0 ms 2024-11-07T14:21:14,594 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:14,594 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6fbbea4d395a3aec7237b0e5e1ceaae5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:14,610 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/.tmp/testFamily/ba1169e7434e4f1cb79dc11bbe84b1a8 is 61, key is row-0/testFamily:testQualifier/1730989201144/Put/seqid=0 2024-11-07T14:21:14,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741903_1079 (size=5049) 2024-11-07T14:21:14,619 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:14,819 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:15,015 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/.tmp/testFamily/ba1169e7434e4f1cb79dc11bbe84b1a8 2024-11-07T14:21:15,020 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-07T14:21:15,020 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 6fbbea4d395a3aec7237b0e5e1ceaae5, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:15,023 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/.tmp/testFamily/ba1169e7434e4f1cb79dc11bbe84b1a8 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/testFamily/ba1169e7434e4f1cb79dc11bbe84b1a8 2024-11-07T14:21:15,030 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/testFamily/ba1169e7434e4f1cb79dc11bbe84b1a8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:15,031 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6fbbea4d395a3aec7237b0e5e1ceaae5 in 437ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:15,036 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_1_-true/6fbbea4d395a3aec7237b0e5e1ceaae5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:15,037 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:15,037 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6fbbea4d395a3aec7237b0e5e1ceaae5: 2024-11-07T14:21:15,037 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1730989200120.6fbbea4d395a3aec7237b0e5e1ceaae5. 2024-11-07T14:21:15,037 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing fcb6eac7f983513750598f0dd2c593b3, disabling compactions & flushes 2024-11-07T14:21:15,037 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:21:15,037 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:21:15,037 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. after waiting 0 ms 2024-11-07T14:21:15,037 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:21:15,037 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing fcb6eac7f983513750598f0dd2c593b3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:15,053 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/.tmp/testFamily/616530b25f104ede88cd50b6e1711cc0 is 61, key is row-0/testFamily:testQualifier/1730989094145/Put/seqid=0 2024-11-07T14:21:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741904_1080 (size=5049) 2024-11-07T14:21:15,220 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:15,420 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c, fcb6eac7f983513750598f0dd2c593b3 2024-11-07T14:21:15,459 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/.tmp/testFamily/616530b25f104ede88cd50b6e1711cc0 2024-11-07T14:21:15,466 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/.tmp/testFamily/616530b25f104ede88cd50b6e1711cc0 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/testFamily/616530b25f104ede88cd50b6e1711cc0 2024-11-07T14:21:15,472 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/testFamily/616530b25f104ede88cd50b6e1711cc0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:15,473 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fcb6eac7f983513750598f0dd2c593b3 in 436ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:15,477 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fcb6eac7f983513750598f0dd2c593b3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:15,478 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:21:15,478 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for fcb6eac7f983513750598f0dd2c593b3: 2024-11-07T14:21:15,478 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730989093096.fcb6eac7f983513750598f0dd2c593b3. 2024-11-07T14:21:15,478 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bf49605def8c8556fa53cbe805f31c9f, disabling compactions & flushes 2024-11-07T14:21:15,478 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:21:15,478 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:21:15,478 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. after waiting 0 ms 2024-11-07T14:21:15,478 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:21:15,478 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bf49605def8c8556fa53cbe805f31c9f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:15,494 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/.tmp/testFamily/938f9283f77a4b15947bf245ef0645c2 is 61, key is row-0/testFamily:testQualifier/1730989209474/Put/seqid=0 2024-11-07T14:21:15,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741905_1081 (size=5049) 2024-11-07T14:21:15,621 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:15,821 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, bf49605def8c8556fa53cbe805f31c9f, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:15,899 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/.tmp/testFamily/938f9283f77a4b15947bf245ef0645c2 2024-11-07T14:21:15,906 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/.tmp/testFamily/938f9283f77a4b15947bf245ef0645c2 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/testFamily/938f9283f77a4b15947bf245ef0645c2 2024-11-07T14:21:15,912 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/testFamily/938f9283f77a4b15947bf245ef0645c2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:15,913 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bf49605def8c8556fa53cbe805f31c9f in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:15,916 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bf49605def8c8556fa53cbe805f31c9f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:15,917 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:21:15,917 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bf49605def8c8556fa53cbe805f31c9f: 2024-11-07T14:21:15,917 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1730989208454.bf49605def8c8556fa53cbe805f31c9f. 2024-11-07T14:21:15,917 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0b2bb17d0bed0d5b693dc62e7f6bbc01, disabling compactions & flushes 2024-11-07T14:21:15,917 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:21:15,917 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:21:15,917 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. after waiting 0 ms 2024-11-07T14:21:15,917 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:21:15,917 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0b2bb17d0bed0d5b693dc62e7f6bbc01 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:15,933 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/.tmp/testFamily/2f963fb7a3bb41a18f3293334b3e1ae2 is 61, key is row-0/testFamily:testQualifier/1730989213551/Put/seqid=0 2024-11-07T14:21:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741906_1082 (size=5049) 2024-11-07T14:21:16,021 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 18 regions to close 2024-11-07T14:21:16,021 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:16,221 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0b2bb17d0bed0d5b693dc62e7f6bbc01, 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:16,338 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/.tmp/testFamily/2f963fb7a3bb41a18f3293334b3e1ae2 2024-11-07T14:21:16,345 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/.tmp/testFamily/2f963fb7a3bb41a18f3293334b3e1ae2 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/testFamily/2f963fb7a3bb41a18f3293334b3e1ae2 2024-11-07T14:21:16,352 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/testFamily/2f963fb7a3bb41a18f3293334b3e1ae2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:16,353 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0b2bb17d0bed0d5b693dc62e7f6bbc01 in 436ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:16,360 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b2bb17d0bed0d5b693dc62e7f6bbc01/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:16,361 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:21:16,361 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0b2bb17d0bed0d5b693dc62e7f6bbc01: 2024-11-07T14:21:16,361 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1730989212532.0b2bb17d0bed0d5b693dc62e7f6bbc01. 2024-11-07T14:21:16,361 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4f1f1bdcd9e70c414da10881c0097694, disabling compactions & flushes 2024-11-07T14:21:16,361 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:21:16,361 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:21:16,361 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. after waiting 0 ms 2024-11-07T14:21:16,361 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:21:16,361 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4f1f1bdcd9e70c414da10881c0097694 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:16,384 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/.tmp/testFamily/96d9bc0504404fd2a3cfb5c20f2157c4 is 61, key is row-0/testFamily:testQualifier/1730989118837/Put/seqid=0 2024-11-07T14:21:16,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741907_1083 (size=5049) 2024-11-07T14:21:16,422 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:16,622 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 4f1f1bdcd9e70c414da10881c0097694, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:16,790 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/.tmp/testFamily/96d9bc0504404fd2a3cfb5c20f2157c4 2024-11-07T14:21:16,797 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/.tmp/testFamily/96d9bc0504404fd2a3cfb5c20f2157c4 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/testFamily/96d9bc0504404fd2a3cfb5c20f2157c4 2024-11-07T14:21:16,803 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/testFamily/96d9bc0504404fd2a3cfb5c20f2157c4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:16,803 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4f1f1bdcd9e70c414da10881c0097694 in 442ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:16,808 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnRenewLease_0_-false/4f1f1bdcd9e70c414da10881c0097694/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:16,808 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:21:16,808 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4f1f1bdcd9e70c414da10881c0097694: 2024-11-07T14:21:16,808 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730989117800.4f1f1bdcd9e70c414da10881c0097694. 2024-11-07T14:21:16,808 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 93cfb54d75f976bc77e2ff0d4c799e55, disabling compactions & flushes 2024-11-07T14:21:16,808 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:21:16,809 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:21:16,809 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. after waiting 0 ms 2024-11-07T14:21:16,809 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:21:16,809 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 93cfb54d75f976bc77e2ff0d4c799e55 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:16,822 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:16,824 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/.tmp/testFamily/3171b2fc316e49cdb299a81e75f45249 is 61, key is row-0/testFamily:testQualifier/1730989154615/Put/seqid=0 2024-11-07T14:21:16,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741908_1084 (size=5049) 2024-11-07T14:21:17,009 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:21:17,022 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-07T14:21:17,023 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:17,223 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, 93cfb54d75f976bc77e2ff0d4c799e55, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:17,229 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/.tmp/testFamily/3171b2fc316e49cdb299a81e75f45249 2024-11-07T14:21:17,236 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/.tmp/testFamily/3171b2fc316e49cdb299a81e75f45249 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/testFamily/3171b2fc316e49cdb299a81e75f45249 2024-11-07T14:21:17,242 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/testFamily/3171b2fc316e49cdb299a81e75f45249, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:17,243 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 93cfb54d75f976bc77e2ff0d4c799e55 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:17,246 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/93cfb54d75f976bc77e2ff0d4c799e55/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:17,247 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:21:17,247 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 93cfb54d75f976bc77e2ff0d4c799e55: 2024-11-07T14:21:17,247 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730989153594.93cfb54d75f976bc77e2ff0d4c799e55. 2024-11-07T14:21:17,247 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing aeb7f5460c70a1472f020d629c072193, disabling compactions & flushes 2024-11-07T14:21:17,248 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:21:17,248 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:21:17,248 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. after waiting 0 ms 2024-11-07T14:21:17,248 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:21:17,248 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing aeb7f5460c70a1472f020d629c072193 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:17,263 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/.tmp/testFamily/4e9c0b5fdae04709bbb6d7707a2b552b is 61, key is row-0/testFamily:testQualifier/1730989255921/Put/seqid=0 2024-11-07T14:21:17,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741909_1085 (size=5049) 2024-11-07T14:21:17,423 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:17,623 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, aeb7f5460c70a1472f020d629c072193, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:17,668 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/.tmp/testFamily/4e9c0b5fdae04709bbb6d7707a2b552b 2024-11-07T14:21:17,676 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/.tmp/testFamily/4e9c0b5fdae04709bbb6d7707a2b552b as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/testFamily/4e9c0b5fdae04709bbb6d7707a2b552b 2024-11-07T14:21:17,682 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/testFamily/4e9c0b5fdae04709bbb6d7707a2b552b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:17,683 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aeb7f5460c70a1472f020d629c072193 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:17,687 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_1_-true/aeb7f5460c70a1472f020d629c072193/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:17,688 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:21:17,688 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for aeb7f5460c70a1472f020d629c072193: 2024-11-07T14:21:17,688 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1730989252906.aeb7f5460c70a1472f020d629c072193. 2024-11-07T14:21:17,688 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1db1b29a17977aa78044ceac0b9bb46e, disabling compactions & flushes 2024-11-07T14:21:17,688 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:21:17,688 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:21:17,688 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. after waiting 0 ms 2024-11-07T14:21:17,688 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:21:17,688 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1db1b29a17977aa78044ceac0b9bb46e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:17,704 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/.tmp/testFamily/c646207458ff4160a3bd38f1f9453f20 is 61, key is row-0/testFamily:testQualifier/1730989223112/Put/seqid=0 2024-11-07T14:21:17,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741910_1086 (size=5049) 2024-11-07T14:21:17,824 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:18,024 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 14 regions to close 2024-11-07T14:21:18,024 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 1db1b29a17977aa78044ceac0b9bb46e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:18,109 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/.tmp/testFamily/c646207458ff4160a3bd38f1f9453f20 2024-11-07T14:21:18,116 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/.tmp/testFamily/c646207458ff4160a3bd38f1f9453f20 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/testFamily/c646207458ff4160a3bd38f1f9453f20 2024-11-07T14:21:18,122 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/testFamily/c646207458ff4160a3bd38f1f9453f20, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:18,122 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1db1b29a17977aa78044ceac0b9bb46e in 434ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:18,126 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_1_-true/1db1b29a17977aa78044ceac0b9bb46e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:18,127 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:21:18,127 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1db1b29a17977aa78044ceac0b9bb46e: 2024-11-07T14:21:18,127 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1730989220094.1db1b29a17977aa78044ceac0b9bb46e. 2024-11-07T14:21:18,127 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0e797cba1bb85ec7f7b6bb605c8e5361, disabling compactions & flushes 2024-11-07T14:21:18,127 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:21:18,127 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:21:18,127 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. after waiting 0 ms 2024-11-07T14:21:18,127 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:21:18,128 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0e797cba1bb85ec7f7b6bb605c8e5361 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:18,143 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/.tmp/testFamily/9f8b2f0754e7467f8a9ddcabdb10777b is 61, key is row-0/testFamily:testQualifier/1730989117651/Put/seqid=0 2024-11-07T14:21:18,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741911_1087 (size=5049) 2024-11-07T14:21:18,224 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:18,424 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 0e797cba1bb85ec7f7b6bb605c8e5361, 17f066662d05461282843a956500594e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:18,549 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/.tmp/testFamily/9f8b2f0754e7467f8a9ddcabdb10777b 2024-11-07T14:21:18,555 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/.tmp/testFamily/9f8b2f0754e7467f8a9ddcabdb10777b as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/testFamily/9f8b2f0754e7467f8a9ddcabdb10777b 2024-11-07T14:21:18,561 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/testFamily/9f8b2f0754e7467f8a9ddcabdb10777b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:18,562 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0e797cba1bb85ec7f7b6bb605c8e5361 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:18,566 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextException_0_-false/0e797cba1bb85ec7f7b6bb605c8e5361/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:18,567 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:21:18,567 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0e797cba1bb85ec7f7b6bb605c8e5361: 2024-11-07T14:21:18,567 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730989116598.0e797cba1bb85ec7f7b6bb605c8e5361. 2024-11-07T14:21:18,567 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 17f066662d05461282843a956500594e, disabling compactions & flushes 2024-11-07T14:21:18,567 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:21:18,567 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:21:18,567 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. after waiting 0 ms 2024-11-07T14:21:18,567 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:21:18,567 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 17f066662d05461282843a956500594e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:18,583 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/.tmp/testFamily/3328d22a82dd4911a99b14e22269b17b is 61, key is row-0/testFamily:testQualifier/1730989243857/Put/seqid=0 2024-11-07T14:21:18,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741912_1088 (size=5049) 2024-11-07T14:21:18,625 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 17f066662d05461282843a956500594e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:18,825 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 17f066662d05461282843a956500594e, 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T14:21:18,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T14:21:18,988 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/.tmp/testFamily/3328d22a82dd4911a99b14e22269b17b 2024-11-07T14:21:18,996 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/.tmp/testFamily/3328d22a82dd4911a99b14e22269b17b as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/testFamily/3328d22a82dd4911a99b14e22269b17b 2024-11-07T14:21:19,002 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/testFamily/3328d22a82dd4911a99b14e22269b17b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:19,003 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17f066662d05461282843a956500594e in 436ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:19,007 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/17f066662d05461282843a956500594e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:19,008 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:21:19,008 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 17f066662d05461282843a956500594e: 2024-11-07T14:21:19,008 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730989240841.17f066662d05461282843a956500594e. 2024-11-07T14:21:19,008 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f292f0dddc0193d61d9f927f4dc3af4c, disabling compactions & flushes 2024-11-07T14:21:19,008 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:21:19,008 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:21:19,008 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. after waiting 0 ms 2024-11-07T14:21:19,009 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:21:19,009 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f292f0dddc0193d61d9f927f4dc3af4c 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-07T14:21:19,025 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/.tmp/info/e391bfb77f8b4312a01934317af77eb1 is 45, key is default/info:d/1730989092877/Put/seqid=0 2024-11-07T14:21:19,025 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 11 regions to close 2024-11-07T14:21:19,025 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:19,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741913_1089 (size=5037) 2024-11-07T14:21:19,226 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:19,426 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40, f292f0dddc0193d61d9f927f4dc3af4c 2024-11-07T14:21:19,430 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/.tmp/info/e391bfb77f8b4312a01934317af77eb1 2024-11-07T14:21:19,437 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/.tmp/info/e391bfb77f8b4312a01934317af77eb1 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/info/e391bfb77f8b4312a01934317af77eb1 2024-11-07T14:21:19,444 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/info/e391bfb77f8b4312a01934317af77eb1, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T14:21:19,445 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for f292f0dddc0193d61d9f927f4dc3af4c in 436ms, sequenceid=6, compaction requested=false 2024-11-07T14:21:19,449 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/namespace/f292f0dddc0193d61d9f927f4dc3af4c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T14:21:19,450 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:21:19,450 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f292f0dddc0193d61d9f927f4dc3af4c: 2024-11-07T14:21:19,450 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1730989092315.f292f0dddc0193d61d9f927f4dc3af4c. 2024-11-07T14:21:19,450 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b6f9ffe8865cf5861efa2412fd098b17, disabling compactions & flushes 2024-11-07T14:21:19,450 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:21:19,450 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:21:19,450 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. after waiting 0 ms 2024-11-07T14:21:19,450 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:21:19,450 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b6f9ffe8865cf5861efa2412fd098b17 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:19,466 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/.tmp/testFamily/fcaf4ed5af56484b88267bd7817a58a3 is 61, key is row-0/testFamily:testQualifier/1730989128009/Put/seqid=0 2024-11-07T14:21:19,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741914_1090 (size=5049) 2024-11-07T14:21:19,626 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:19,826 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b6f9ffe8865cf5861efa2412fd098b17, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:19,871 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/.tmp/testFamily/fcaf4ed5af56484b88267bd7817a58a3 2024-11-07T14:21:19,878 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/.tmp/testFamily/fcaf4ed5af56484b88267bd7817a58a3 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/testFamily/fcaf4ed5af56484b88267bd7817a58a3 2024-11-07T14:21:19,884 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/testFamily/fcaf4ed5af56484b88267bd7817a58a3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:19,885 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b6f9ffe8865cf5861efa2412fd098b17 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:19,889 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b6f9ffe8865cf5861efa2412fd098b17/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:19,889 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:21:19,889 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b6f9ffe8865cf5861efa2412fd098b17: 2024-11-07T14:21:19,889 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730989124987.b6f9ffe8865cf5861efa2412fd098b17. 2024-11-07T14:21:19,890 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d04a9d74f6b94963a147e616bf79af65, disabling compactions & flushes 2024-11-07T14:21:19,890 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:21:19,890 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:21:19,890 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. after waiting 0 ms 2024-11-07T14:21:19,890 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:21:19,890 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d04a9d74f6b94963a147e616bf79af65 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:19,905 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/.tmp/testFamily/f02aa5554b64491a9543cc108307b351 is 61, key is row-0/testFamily:testQualifier/1730989191063/Put/seqid=0 2024-11-07T14:21:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741915_1091 (size=5049) 2024-11-07T14:21:20,027 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 9 regions to close 2024-11-07T14:21:20,027 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1603): Online Regions={d04a9d74f6b94963a147e616bf79af65=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65., ba1a3fb142dd50d2b08b0509b442a780=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780., 7ebdf008998e34da6a8bfd565c333c77=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., ebd705d1230cccee05e70a4866f70c40=hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40., 8e7403a5d1cd160a13036a66684da366=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., 6c03090e181c03a5de5603313432cd9a=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., b93ccd845b07475bea616303fbd6a1a6=testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6., 6db8379cd8b72d893208d9bc43753e66=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66., c6d73ad91ef7cb6fc0537617e72d4e34=hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.} 2024-11-07T14:21:20,027 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:20,227 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, d04a9d74f6b94963a147e616bf79af65, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:20,310 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/.tmp/testFamily/f02aa5554b64491a9543cc108307b351 2024-11-07T14:21:20,316 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/.tmp/testFamily/f02aa5554b64491a9543cc108307b351 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/testFamily/f02aa5554b64491a9543cc108307b351 2024-11-07T14:21:20,322 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/testFamily/f02aa5554b64491a9543cc108307b351, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:20,323 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d04a9d74f6b94963a147e616bf79af65 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:20,326 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/d04a9d74f6b94963a147e616bf79af65/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:20,327 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:21:20,327 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d04a9d74f6b94963a147e616bf79af65: 2024-11-07T14:21:20,327 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730989188046.d04a9d74f6b94963a147e616bf79af65. 2024-11-07T14:21:20,327 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ba1a3fb142dd50d2b08b0509b442a780, disabling compactions & flushes 2024-11-07T14:21:20,327 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:21:20,327 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:21:20,327 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. after waiting 0 ms 2024-11-07T14:21:20,327 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:21:20,327 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ba1a3fb142dd50d2b08b0509b442a780 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:20,342 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/.tmp/testFamily/a089ba1b5425450bb135612c17459390 is 61, key is row-0/testFamily:testQualifier/1730989233787/Put/seqid=0 2024-11-07T14:21:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741916_1092 (size=5049) 2024-11-07T14:21:20,427 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:20,628 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, ba1a3fb142dd50d2b08b0509b442a780, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:20,747 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/.tmp/testFamily/a089ba1b5425450bb135612c17459390 2024-11-07T14:21:20,755 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/.tmp/testFamily/a089ba1b5425450bb135612c17459390 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/testFamily/a089ba1b5425450bb135612c17459390 2024-11-07T14:21:20,762 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/testFamily/a089ba1b5425450bb135612c17459390, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:20,763 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ba1a3fb142dd50d2b08b0509b442a780 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:20,767 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/ba1a3fb142dd50d2b08b0509b442a780/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:20,767 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:21:20,767 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ba1a3fb142dd50d2b08b0509b442a780: 2024-11-07T14:21:20,767 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730989230751.ba1a3fb142dd50d2b08b0509b442a780. 2024-11-07T14:21:20,768 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7ebdf008998e34da6a8bfd565c333c77, disabling compactions & flushes 2024-11-07T14:21:20,768 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:20,768 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:20,768 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. after waiting 0 ms 2024-11-07T14:21:20,768 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:20,768 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7ebdf008998e34da6a8bfd565c333c77 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:20,786 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/.tmp/testFamily/4a99fc1c3cb44af6ba7f315659204dac is 61, key is row-0/testFamily:testQualifier/1730989205374/Put/seqid=0 2024-11-07T14:21:20,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741917_1093 (size=5049) 2024-11-07T14:21:20,828 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:21,028 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 7 regions to close 2024-11-07T14:21:21,028 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1603): Online Regions={7ebdf008998e34da6a8bfd565c333c77=testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77., ebd705d1230cccee05e70a4866f70c40=hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40., 8e7403a5d1cd160a13036a66684da366=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., 6c03090e181c03a5de5603313432cd9a=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., b93ccd845b07475bea616303fbd6a1a6=testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6., 6db8379cd8b72d893208d9bc43753e66=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66., c6d73ad91ef7cb6fc0537617e72d4e34=hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.} 2024-11-07T14:21:21,028 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 7ebdf008998e34da6a8bfd565c333c77, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:21,192 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/.tmp/testFamily/4a99fc1c3cb44af6ba7f315659204dac 2024-11-07T14:21:21,198 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/.tmp/testFamily/4a99fc1c3cb44af6ba7f315659204dac as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/testFamily/4a99fc1c3cb44af6ba7f315659204dac 2024-11-07T14:21:21,204 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/testFamily/4a99fc1c3cb44af6ba7f315659204dac, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:21,205 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ebdf008998e34da6a8bfd565c333c77 in 437ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:21,209 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/7ebdf008998e34da6a8bfd565c333c77/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:21,209 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:21,209 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7ebdf008998e34da6a8bfd565c333c77: 2024-11-07T14:21:21,209 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1730989204338.7ebdf008998e34da6a8bfd565c333c77. 2024-11-07T14:21:21,209 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ebd705d1230cccee05e70a4866f70c40, disabling compactions & flushes 2024-11-07T14:21:21,209 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:21:21,209 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:21:21,209 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. after waiting 0 ms 2024-11-07T14:21:21,210 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:21:21,210 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ebd705d1230cccee05e70a4866f70c40 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:21,225 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/.tmp/testFamily/a77fa8f5f04e49aea543477adca1a93f is 61, key is row-0/testFamily:testQualifier/1730989134087/Put/seqid=0 2024-11-07T14:21:21,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741918_1094 (size=5049) 2024-11-07T14:21:21,228 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:21,429 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:21,629 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34, ebd705d1230cccee05e70a4866f70c40 2024-11-07T14:21:21,629 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/.tmp/testFamily/a77fa8f5f04e49aea543477adca1a93f 2024-11-07T14:21:21,635 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/.tmp/testFamily/a77fa8f5f04e49aea543477adca1a93f as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/testFamily/a77fa8f5f04e49aea543477adca1a93f 2024-11-07T14:21:21,640 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/testFamily/a77fa8f5f04e49aea543477adca1a93f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:21,641 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ebd705d1230cccee05e70a4866f70c40 in 431ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:21,645 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnClose_0_-false/ebd705d1230cccee05e70a4866f70c40/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:21,646 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:21:21,646 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ebd705d1230cccee05e70a4866f70c40: 2024-11-07T14:21:21,646 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730989131066.ebd705d1230cccee05e70a4866f70c40. 2024-11-07T14:21:21,646 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8e7403a5d1cd160a13036a66684da366, disabling compactions & flushes 2024-11-07T14:21:21,646 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:21:21,646 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:21:21,646 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. after waiting 0 ms 2024-11-07T14:21:21,646 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:21:21,646 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8e7403a5d1cd160a13036a66684da366 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:21,661 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/.tmp/testFamily/ff732e1ca488440ea11e16f9787a2de0 is 61, key is row-0/testFamily:testQualifier/1730989179001/Put/seqid=0 2024-11-07T14:21:21,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741919_1095 (size=5049) 2024-11-07T14:21:21,829 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:22,029 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 5 regions to close 2024-11-07T14:21:22,030 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1603): Online Regions={8e7403a5d1cd160a13036a66684da366=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366., 6c03090e181c03a5de5603313432cd9a=testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a., b93ccd845b07475bea616303fbd6a1a6=testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6., 6db8379cd8b72d893208d9bc43753e66=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66., c6d73ad91ef7cb6fc0537617e72d4e34=hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.} 2024-11-07T14:21:22,030 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, 8e7403a5d1cd160a13036a66684da366, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:22,067 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/.tmp/testFamily/ff732e1ca488440ea11e16f9787a2de0 2024-11-07T14:21:22,073 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/.tmp/testFamily/ff732e1ca488440ea11e16f9787a2de0 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/testFamily/ff732e1ca488440ea11e16f9787a2de0 2024-11-07T14:21:22,079 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/testFamily/ff732e1ca488440ea11e16f9787a2de0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:22,080 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8e7403a5d1cd160a13036a66684da366 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:22,084 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/8e7403a5d1cd160a13036a66684da366/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:22,084 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:21:22,084 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8e7403a5d1cd160a13036a66684da366: 2024-11-07T14:21:22,085 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730989177980.8e7403a5d1cd160a13036a66684da366. 2024-11-07T14:21:22,085 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6c03090e181c03a5de5603313432cd9a, disabling compactions & flushes 2024-11-07T14:21:22,085 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:21:22,085 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:21:22,085 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. after waiting 0 ms 2024-11-07T14:21:22,085 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:21:22,085 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6c03090e181c03a5de5603313432cd9a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:22,100 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/.tmp/testFamily/492ead20afd54a89bd2cfc1c56ed8f9c is 61, key is row-0/testFamily:testQualifier/1730989121937/Put/seqid=0 2024-11-07T14:21:22,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741920_1096 (size=5049) 2024-11-07T14:21:22,230 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:22,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:21:22,430 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6c03090e181c03a5de5603313432cd9a, 6db8379cd8b72d893208d9bc43753e66, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:22,505 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/.tmp/testFamily/492ead20afd54a89bd2cfc1c56ed8f9c 2024-11-07T14:21:22,512 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/.tmp/testFamily/492ead20afd54a89bd2cfc1c56ed8f9c as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/testFamily/492ead20afd54a89bd2cfc1c56ed8f9c 2024-11-07T14:21:22,517 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/testFamily/492ead20afd54a89bd2cfc1c56ed8f9c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:22,518 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c03090e181c03a5de5603313432cd9a in 433ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:22,521 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6c03090e181c03a5de5603313432cd9a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:22,522 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:21:22,522 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6c03090e181c03a5de5603313432cd9a: 2024-11-07T14:21:22,522 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730989120911.6c03090e181c03a5de5603313432cd9a. 2024-11-07T14:21:22,522 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b93ccd845b07475bea616303fbd6a1a6, disabling compactions & flushes 2024-11-07T14:21:22,522 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:21:22,522 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:21:22,522 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. after waiting 0 ms 2024-11-07T14:21:22,522 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:21:22,522 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b93ccd845b07475bea616303fbd6a1a6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:22,537 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/.tmp/testFamily/c3304d3aca344d2a8a5797242658bd06 is 61, key is row-0/testFamily:testQualifier/1730989141692/Put/seqid=0 2024-11-07T14:21:22,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741921_1097 (size=5049) 2024-11-07T14:21:22,630 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6db8379cd8b72d893208d9bc43753e66, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:22,831 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6db8379cd8b72d893208d9bc43753e66, b93ccd845b07475bea616303fbd6a1a6, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:22,942 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/.tmp/testFamily/c3304d3aca344d2a8a5797242658bd06 2024-11-07T14:21:22,949 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/.tmp/testFamily/c3304d3aca344d2a8a5797242658bd06 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/testFamily/c3304d3aca344d2a8a5797242658bd06 2024-11-07T14:21:22,954 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/testFamily/c3304d3aca344d2a8a5797242658bd06, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:22,955 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b93ccd845b07475bea616303fbd6a1a6 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:22,960 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testNormalScanTimeoutOnClose_0_-false/b93ccd845b07475bea616303fbd6a1a6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:22,961 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:21:22,961 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b93ccd845b07475bea616303fbd6a1a6: 2024-11-07T14:21:22,961 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1730989140652.b93ccd845b07475bea616303fbd6a1a6. 2024-11-07T14:21:22,961 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6db8379cd8b72d893208d9bc43753e66, disabling compactions & flushes 2024-11-07T14:21:22,961 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:21:22,961 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:21:22,961 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. after waiting 0 ms 2024-11-07T14:21:22,961 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:21:22,961 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6db8379cd8b72d893208d9bc43753e66 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:22,977 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/.tmp/testFamily/3f029a5227ab46e5803999fc7818a240 is 61, key is row-0/testFamily:testQualifier/1730989177896/Put/seqid=0 2024-11-07T14:21:22,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741922_1098 (size=5049) 2024-11-07T14:21:23,031 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-07T14:21:23,031 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1603): Online Regions={6db8379cd8b72d893208d9bc43753e66=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66., c6d73ad91ef7cb6fc0537617e72d4e34=hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34.} 2024-11-07T14:21:23,031 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6db8379cd8b72d893208d9bc43753e66, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:23,231 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on 6db8379cd8b72d893208d9bc43753e66, c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:23,387 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/.tmp/testFamily/3f029a5227ab46e5803999fc7818a240 2024-11-07T14:21:23,394 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/.tmp/testFamily/3f029a5227ab46e5803999fc7818a240 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/testFamily/3f029a5227ab46e5803999fc7818a240 2024-11-07T14:21:23,399 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/testFamily/3f029a5227ab46e5803999fc7818a240, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:23,400 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6db8379cd8b72d893208d9bc43753e66 in 439ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:23,404 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6db8379cd8b72d893208d9bc43753e66/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:23,405 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:21:23,405 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6db8379cd8b72d893208d9bc43753e66: 2024-11-07T14:21:23,405 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730989176878.6db8379cd8b72d893208d9bc43753e66. 2024-11-07T14:21:23,405 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c6d73ad91ef7cb6fc0537617e72d4e34, disabling compactions & flushes 2024-11-07T14:21:23,405 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:21:23,405 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:21:23,405 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. after waiting 0 ms 2024-11-07T14:21:23,405 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:21:23,405 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c6d73ad91ef7cb6fc0537617e72d4e34 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T14:21:23,420 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/.tmp/testFamily/84104030b52946e394972ae2991654d3 is 61, key is row-0/testFamily:testQualifier/1730989166688/Put/seqid=0 2024-11-07T14:21:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741923_1099 (size=5049) 2024-11-07T14:21:23,431 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:23,632 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:23,825 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/.tmp/testFamily/84104030b52946e394972ae2991654d3 2024-11-07T14:21:23,831 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/.tmp/testFamily/84104030b52946e394972ae2991654d3 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/testFamily/84104030b52946e394972ae2991654d3 2024-11-07T14:21:23,832 DEBUG [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1629): Waiting on c6d73ad91ef7cb6fc0537617e72d4e34 2024-11-07T14:21:23,836 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/testFamily/84104030b52946e394972ae2991654d3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T14:21:23,837 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c6d73ad91ef7cb6fc0537617e72d4e34 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T14:21:23,841 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/data/hbase/testMetaScanTimeoutOnNext_0_-false/c6d73ad91ef7cb6fc0537617e72d4e34/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T14:21:23,842 INFO [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:21:23,842 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c6d73ad91ef7cb6fc0537617e72d4e34: 2024-11-07T14:21:23,842 DEBUG [RS_CLOSE_REGION-regionserver/69430dbfd73f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730989163666.c6d73ad91ef7cb6fc0537617e72d4e34. 2024-11-07T14:21:24,032 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1250): stopping server 69430dbfd73f,36593,1730989059679; all regions closed. 2024-11-07T14:21:24,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741834_1010 (size=56731) 2024-11-07T14:21:24,042 DEBUG [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/oldWALs 2024-11-07T14:21:24,042 INFO [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 69430dbfd73f%2C36593%2C1730989059679.meta:.meta(num 1730989092012) 2024-11-07T14:21:24,045 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/WALs/69430dbfd73f,36593,1730989059679/69430dbfd73f%2C36593%2C1730989059679.1730989091725 not finished, retry = 0 2024-11-07T14:21:24,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741833_1009 (size=60562) 2024-11-07T14:21:24,148 DEBUG [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/oldWALs 2024-11-07T14:21:24,148 INFO [RS:0;69430dbfd73f:36593 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 69430dbfd73f%2C36593%2C1730989059679:(num 1730989091725) 2024-11-07T14:21:24,148 DEBUG [RS:0;69430dbfd73f:36593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:24,148 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T14:21:24,149 INFO [RS:0;69430dbfd73f:36593 {}] hbase.ChoreService(370): Chore service for: regionserver/69430dbfd73f:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T14:21:24,149 INFO [regionserver/69430dbfd73f:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T14:21:24,150 INFO [RS:0;69430dbfd73f:36593 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36593 2024-11-07T14:21:24,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/69430dbfd73f,36593,1730989059679 2024-11-07T14:21:24,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T14:21:24,157 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [69430dbfd73f,36593,1730989059679] 2024-11-07T14:21:24,157 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 69430dbfd73f,36593,1730989059679; numProcessing=1 2024-11-07T14:21:24,159 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/69430dbfd73f,36593,1730989059679 already deleted, retry=false 2024-11-07T14:21:24,159 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 69430dbfd73f,36593,1730989059679 expired; onlineServers=0 2024-11-07T14:21:24,159 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '69430dbfd73f,42745,1730989058075' ***** 2024-11-07T14:21:24,159 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T14:21:24,159 DEBUG [M:0;69430dbfd73f:42745 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e75eb4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=69430dbfd73f/172.17.0.2:0 2024-11-07T14:21:24,159 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegionServer(1224): stopping server 69430dbfd73f,42745,1730989058075 2024-11-07T14:21:24,159 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegionServer(1250): stopping server 69430dbfd73f,42745,1730989058075; all regions closed. 2024-11-07T14:21:24,159 DEBUG [M:0;69430dbfd73f:42745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:21:24,159 DEBUG [M:0;69430dbfd73f:42745 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T14:21:24,159 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T14:21:24,159 DEBUG [M:0;69430dbfd73f:42745 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T14:21:24,159 DEBUG [master/69430dbfd73f:0:becomeActiveMaster-HFileCleaner.large.0-1730989061414 {}] cleaner.HFileCleaner(306): Exit Thread[master/69430dbfd73f:0:becomeActiveMaster-HFileCleaner.large.0-1730989061414,5,FailOnTimeoutGroup] 2024-11-07T14:21:24,159 DEBUG [master/69430dbfd73f:0:becomeActiveMaster-HFileCleaner.small.0-1730989061421 {}] cleaner.HFileCleaner(306): Exit Thread[master/69430dbfd73f:0:becomeActiveMaster-HFileCleaner.small.0-1730989061421,5,FailOnTimeoutGroup] 2024-11-07T14:21:24,160 INFO [M:0;69430dbfd73f:42745 {}] hbase.ChoreService(370): Chore service for: master/69430dbfd73f:0 had [] on shutdown 2024-11-07T14:21:24,160 DEBUG [M:0;69430dbfd73f:42745 {}] master.HMaster(1733): Stopping service threads 2024-11-07T14:21:24,160 INFO [M:0;69430dbfd73f:42745 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T14:21:24,160 ERROR [M:0;69430dbfd73f:42745 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[LeaseRenewer:jenkins@localhost:41411,5,PEWorkerGroup] 2024-11-07T14:21:24,160 INFO [M:0;69430dbfd73f:42745 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T14:21:24,161 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T14:21:24,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T14:21:24,161 DEBUG [M:0;69430dbfd73f:42745 {}] zookeeper.ZKUtil(347): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T14:21:24,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:21:24,162 WARN [M:0;69430dbfd73f:42745 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T14:21:24,162 INFO [M:0;69430dbfd73f:42745 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-07T14:21:24,162 INFO [M:0;69430dbfd73f:42745 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T14:21:24,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T14:21:24,162 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T14:21:24,162 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:21:24,162 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:21:24,162 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T14:21:24,162 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:21:24,162 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.42 KB heapSize=519.66 KB 2024-11-07T14:21:24,177 INFO [regionserver/69430dbfd73f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T14:21:24,179 DEBUG [M:0;69430dbfd73f:42745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18b7902c63264e9fb6b24837548bca18 is 82, key is hbase:meta,,1/info:regioninfo/1730989092181/Put/seqid=0 2024-11-07T14:21:24,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741924_1100 (size=5672) 2024-11-07T14:21:24,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:21:24,257 INFO [RS:0;69430dbfd73f:36593 {}] regionserver.HRegionServer(1307): Exiting; stopping=69430dbfd73f,36593,1730989059679; zookeeper connection closed. 2024-11-07T14:21:24,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36593-0x1018318593e0001, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:21:24,257 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24615868 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24615868 2024-11-07T14:21:24,258 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T14:21:24,584 INFO [M:0;69430dbfd73f:42745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=827 (bloomFilter=true), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18b7902c63264e9fb6b24837548bca18 2024-11-07T14:21:24,607 DEBUG [M:0;69430dbfd73f:42745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf7aa62a71b4a27a750d99a4c7f5276 is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1730989093588/Put/seqid=0 2024-11-07T14:21:24,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741925_1101 (size=30291) 2024-11-07T14:21:25,014 INFO [M:0;69430dbfd73f:42745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=447.86 KB at sequenceid=827 (bloomFilter=true), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf7aa62a71b4a27a750d99a4c7f5276 2024-11-07T14:21:25,019 INFO [M:0;69430dbfd73f:42745 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbf7aa62a71b4a27a750d99a4c7f5276 2024-11-07T14:21:25,034 DEBUG [M:0;69430dbfd73f:42745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7747e5363957417ba106dcc5b809fa30 is 69, key is 69430dbfd73f,36593,1730989059679/rs:state/1730989061493/Put/seqid=0 2024-11-07T14:21:25,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741926_1102 (size=5156) 2024-11-07T14:21:25,439 INFO [M:0;69430dbfd73f:42745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=827 (bloomFilter=true), to=hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7747e5363957417ba106dcc5b809fa30 2024-11-07T14:21:25,446 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18b7902c63264e9fb6b24837548bca18 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/18b7902c63264e9fb6b24837548bca18 2024-11-07T14:21:25,451 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/18b7902c63264e9fb6b24837548bca18, entries=8, sequenceid=827, filesize=5.5 K 2024-11-07T14:21:25,452 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf7aa62a71b4a27a750d99a4c7f5276 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbf7aa62a71b4a27a750d99a4c7f5276 2024-11-07T14:21:25,456 INFO [M:0;69430dbfd73f:42745 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbf7aa62a71b4a27a750d99a4c7f5276 2024-11-07T14:21:25,456 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbf7aa62a71b4a27a750d99a4c7f5276, entries=92, sequenceid=827, filesize=29.6 K 2024-11-07T14:21:25,457 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7747e5363957417ba106dcc5b809fa30 as hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7747e5363957417ba106dcc5b809fa30 2024-11-07T14:21:25,461 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41411/user/jenkins/test-data/7077e51c-86c4-66d5-4f18-da24d28c4452/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7747e5363957417ba106dcc5b809fa30, entries=1, sequenceid=827, filesize=5.0 K 2024-11-07T14:21:25,462 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.42 KB/459180, heapSize ~519.37 KB/531832, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1300ms, sequenceid=827, compaction requested=false 2024-11-07T14:21:25,464 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:21:25,464 DEBUG [M:0;69430dbfd73f:42745 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T14:21:25,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37655 is added to blk_1073741830_1006 (size=513854) 2024-11-07T14:21:25,467 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T14:21:25,467 INFO [M:0;69430dbfd73f:42745 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-07T14:21:25,467 INFO [M:0;69430dbfd73f:42745 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42745 2024-11-07T14:21:25,469 DEBUG [M:0;69430dbfd73f:42745 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/69430dbfd73f,42745,1730989058075 already deleted, retry=false 2024-11-07T14:21:25,571 INFO [M:0;69430dbfd73f:42745 {}] regionserver.HRegionServer(1307): Exiting; stopping=69430dbfd73f,42745,1730989058075; zookeeper connection closed. 2024-11-07T14:21:25,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:21:25,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42745-0x1018318593e0000, quorum=127.0.0.1:49795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:21:25,576 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6796851a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T14:21:25,578 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@55c2c7f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T14:21:25,578 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T14:21:25,579 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31b28bed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T14:21:25,579 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@137d3cad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/hadoop.log.dir/,STOPPED} 2024-11-07T14:21:25,581 WARN [BP-730893796-172.17.0.2-1730989052927 heartbeating to localhost/127.0.0.1:41411 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T14:21:25,581 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T14:21:25,581 WARN [BP-730893796-172.17.0.2-1730989052927 heartbeating to localhost/127.0.0.1:41411 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-730893796-172.17.0.2-1730989052927 (Datanode Uuid b9e8e1f7-722c-4be9-9307-8e91f8789de8) service to localhost/127.0.0.1:41411 2024-11-07T14:21:25,581 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T14:21:25,582 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/dfs/data/data1/current/BP-730893796-172.17.0.2-1730989052927 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T14:21:25,583 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/cluster_a7f1c8ca-e242-8ffc-c86c-3495f114f48a/dfs/data/data2/current/BP-730893796-172.17.0.2-1730989052927 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T14:21:25,583 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T14:21:25,592 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27d34e10{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T14:21:25,592 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c62dc53{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T14:21:25,592 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T14:21:25,593 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a37b7ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T14:21:25,593 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@634f7b0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c957b2e2-104e-511e-622a-78e7a2b32e8a/hadoop.log.dir/,STOPPED} 2024-11-07T14:21:25,603 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-07T14:21:25,644 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down