2024-11-09 09:13:18,540 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-09 09:13:18,552 main DEBUG Took 0.010091 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 09:13:18,553 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 09:13:18,553 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 09:13:18,554 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 09:13:18,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,569 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 09:13:18,579 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,581 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,581 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,582 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,583 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,584 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,585 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,586 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,587 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,587 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,589 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,589 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 09:13:18,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,591 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 09:13:18,592 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 09:13:18,593 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 09:13:18,595 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 09:13:18,595 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 09:13:18,596 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 09:13:18,597 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 09:13:18,604 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 09:13:18,607 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 09:13:18,608 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 09:13:18,608 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 09:13:18,609 main DEBUG createAppenders(={Console}) 2024-11-09 09:13:18,610 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-09 09:13:18,610 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-09 09:13:18,610 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-09 09:13:18,611 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 09:13:18,611 main DEBUG OutputStream closed 2024-11-09 09:13:18,611 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 09:13:18,611 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 09:13:18,612 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-09 09:13:18,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 09:13:18,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 09:13:18,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 09:13:18,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 09:13:18,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 09:13:18,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 09:13:18,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 09:13:18,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 09:13:18,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 09:13:18,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 09:13:18,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 09:13:18,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 09:13:18,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 09:13:18,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 09:13:18,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 09:13:18,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 09:13:18,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 09:13:18,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 09:13:18,691 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 09:13:18,692 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-09 09:13:18,692 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 09:13:18,693 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-09T09:13:18,896 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-09 09:13:18,898 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 09:13:18,899 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T09:13:18,961 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b 2024-11-09T09:13:18,992 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T09:13:18,995 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9, deleteOnExit=true 2024-11-09T09:13:18,995 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-09T09:13:18,996 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/test.cache.data in system properties and HBase conf 2024-11-09T09:13:18,996 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T09:13:18,997 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/hadoop.log.dir in system properties and HBase conf 2024-11-09T09:13:18,998 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T09:13:18,998 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T09:13:18,999 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-09T09:13:19,088 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T09:13:19,178 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T09:13:19,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T09:13:19,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T09:13:19,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T09:13:19,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T09:13:19,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T09:13:19,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T09:13:19,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T09:13:19,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T09:13:19,186 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T09:13:19,186 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/nfs.dump.dir in system properties and HBase conf 2024-11-09T09:13:19,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/java.io.tmpdir in system properties and HBase conf 2024-11-09T09:13:19,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T09:13:19,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T09:13:19,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T09:13:20,184 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T09:13:20,255 INFO [Time-limited test {}] log.Log(170): Logging initialized @2352ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T09:13:20,320 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T09:13:20,378 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T09:13:20,398 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T09:13:20,399 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T09:13:20,400 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T09:13:20,412 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T09:13:20,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@301bbbb5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/hadoop.log.dir/,AVAILABLE} 2024-11-09T09:13:20,417 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79eda29b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T09:13:20,589 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@235e3378{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/java.io.tmpdir/jetty-localhost-37319-hadoop-hdfs-3_4_1-tests_jar-_-any-8181031411339970082/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T09:13:20,595 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7481235f{HTTP/1.1, (http/1.1)}{localhost:37319} 2024-11-09T09:13:20,595 INFO [Time-limited test {}] server.Server(415): Started @2693ms 2024-11-09T09:13:21,064 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T09:13:21,070 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T09:13:21,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T09:13:21,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T09:13:21,072 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T09:13:21,072 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29886710{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/hadoop.log.dir/,AVAILABLE} 2024-11-09T09:13:21,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39bf03c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T09:13:21,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4766658a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/java.io.tmpdir/jetty-localhost-37691-hadoop-hdfs-3_4_1-tests_jar-_-any-1992362703299033175/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T09:13:21,171 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@320542b5{HTTP/1.1, (http/1.1)}{localhost:37691} 2024-11-09T09:13:21,171 INFO [Time-limited test {}] server.Server(415): Started @3269ms 2024-11-09T09:13:21,218 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T09:13:22,068 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/dfs/data/data1/current/BP-994757153-172.17.0.2-1731143599737/current, will proceed with Du for space computation calculation, 2024-11-09T09:13:22,068 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/dfs/data/data2/current/BP-994757153-172.17.0.2-1731143599737/current, will proceed with Du for space computation calculation, 2024-11-09T09:13:22,095 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T09:13:22,139 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5296103c60f0f76a with lease ID 0xf6e014c037d84591: Processing first storage report for DS-976a2484-d0ae-4ecb-b005-d5dc8e3bc7b2 from datanode DatanodeRegistration(127.0.0.1:34159, datanodeUuid=4f8332e9-6c27-4623-91d4-6fc003390f36, infoPort=36227, infoSecurePort=0, ipcPort=36105, storageInfo=lv=-57;cid=testClusterID;nsid=177715886;c=1731143599737) 2024-11-09T09:13:22,140 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5296103c60f0f76a with lease ID 0xf6e014c037d84591: from storage DS-976a2484-d0ae-4ecb-b005-d5dc8e3bc7b2 node DatanodeRegistration(127.0.0.1:34159, datanodeUuid=4f8332e9-6c27-4623-91d4-6fc003390f36, infoPort=36227, infoSecurePort=0, ipcPort=36105, storageInfo=lv=-57;cid=testClusterID;nsid=177715886;c=1731143599737), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T09:13:22,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5296103c60f0f76a with lease ID 0xf6e014c037d84591: Processing first storage report for DS-7d2eb6c6-4dfe-4bb9-a732-c0e281f7cc11 from datanode DatanodeRegistration(127.0.0.1:34159, datanodeUuid=4f8332e9-6c27-4623-91d4-6fc003390f36, infoPort=36227, infoSecurePort=0, ipcPort=36105, storageInfo=lv=-57;cid=testClusterID;nsid=177715886;c=1731143599737) 2024-11-09T09:13:22,140 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5296103c60f0f76a with lease ID 0xf6e014c037d84591: from storage DS-7d2eb6c6-4dfe-4bb9-a732-c0e281f7cc11 node DatanodeRegistration(127.0.0.1:34159, datanodeUuid=4f8332e9-6c27-4623-91d4-6fc003390f36, infoPort=36227, infoSecurePort=0, ipcPort=36105, storageInfo=lv=-57;cid=testClusterID;nsid=177715886;c=1731143599737), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T09:13:22,227 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b 2024-11-09T09:13:22,297 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/zookeeper_0, clientPort=49767, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T09:13:22,307 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49767 2024-11-09T09:13:22,317 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:22,319 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:22,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741825_1001 (size=7) 2024-11-09T09:13:22,926 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 with version=8 2024-11-09T09:13:22,927 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/hbase-staging 2024-11-09T09:13:23,031 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T09:13:23,269 INFO [Time-limited test {}] client.ConnectionUtils(129): master/9b029db41fd9:0 server-side Connection retries=9 2024-11-09T09:13:23,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T09:13:23,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T09:13:23,285 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T09:13:23,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T09:13:23,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T09:13:23,400 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T09:13:23,449 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T09:13:23,457 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T09:13:23,460 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T09:13:23,482 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32424 (auto-detected) 2024-11-09T09:13:23,483 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T09:13:23,499 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41847 2024-11-09T09:13:23,505 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:23,507 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:23,518 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:41847 connecting to ZooKeeper ensemble=127.0.0.1:49767 2024-11-09T09:13:23,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418470x0, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T09:13:23,632 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41847-0x1012028b2df0000 connected 2024-11-09T09:13:23,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T09:13:23,707 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T09:13:23,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T09:13:23,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41847 2024-11-09T09:13:23,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41847 2024-11-09T09:13:23,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41847 2024-11-09T09:13:23,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41847 2024-11-09T09:13:23,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41847 2024-11-09T09:13:23,725 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014, hbase.cluster.distributed=false 2024-11-09T09:13:23,788 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/9b029db41fd9:0 server-side Connection retries=9 2024-11-09T09:13:23,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T09:13:23,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T09:13:23,788 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T09:13:23,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T09:13:23,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T09:13:23,790 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T09:13:23,793 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T09:13:23,794 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43449 2024-11-09T09:13:23,795 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T09:13:23,799 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T09:13:23,801 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:23,803 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:23,807 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:43449 connecting to ZooKeeper ensemble=127.0.0.1:49767 2024-11-09T09:13:23,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434490x0, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T09:13:23,817 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434490x0, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T09:13:23,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43449-0x1012028b2df0001 connected 2024-11-09T09:13:23,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T09:13:23,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T09:13:23,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43449 2024-11-09T09:13:23,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43449 2024-11-09T09:13:23,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43449 2024-11-09T09:13:23,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43449 2024-11-09T09:13:23,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43449 2024-11-09T09:13:23,826 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/9b029db41fd9,41847,1731143603026 2024-11-09T09:13:23,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T09:13:23,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T09:13:23,835 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9b029db41fd9,41847,1731143603026 2024-11-09T09:13:23,840 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9b029db41fd9:41847 2024-11-09T09:13:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T09:13:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T09:13:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:23,859 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T09:13:23,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T09:13:23,860 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9b029db41fd9,41847,1731143603026 from backup master directory 2024-11-09T09:13:23,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9b029db41fd9,41847,1731143603026 2024-11-09T09:13:23,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T09:13:23,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T09:13:23,867 WARN [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T09:13:23,867 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9b029db41fd9,41847,1731143603026 2024-11-09T09:13:23,869 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T09:13:23,871 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T09:13:23,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741826_1002 (size=42) 2024-11-09T09:13:23,932 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/hbase.id with ID: 713e361b-88e2-4321-a1ee-2409650bfc38 2024-11-09T09:13:23,977 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T09:13:24,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:24,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:24,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741827_1003 (size=196) 2024-11-09T09:13:24,040 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:13:24,043 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T09:13:24,061 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:13:24,066 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T09:13:24,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741828_1004 (size=1189) 2024-11-09T09:13:24,520 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store 2024-11-09T09:13:24,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741829_1005 (size=34) 2024-11-09T09:13:24,946 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T09:13:24,946 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:24,948 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T09:13:24,948 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:13:24,948 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:13:24,948 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T09:13:24,948 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:13:24,949 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:13:24,949 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T09:13:24,951 WARN [master/9b029db41fd9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/.initializing 2024-11-09T09:13:24,951 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/WALs/9b029db41fd9,41847,1731143603026 2024-11-09T09:13:24,957 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T09:13:24,968 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9b029db41fd9%2C41847%2C1731143603026, suffix=, logDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/WALs/9b029db41fd9,41847,1731143603026, archiveDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/oldWALs, maxLogs=10 2024-11-09T09:13:24,988 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/WALs/9b029db41fd9,41847,1731143603026/9b029db41fd9%2C41847%2C1731143603026.1731143604973, exclude list is [], retry=0 2024-11-09T09:13:25,002 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34159,DS-976a2484-d0ae-4ecb-b005-d5dc8e3bc7b2,DISK] 2024-11-09T09:13:25,004 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T09:13:25,041 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/WALs/9b029db41fd9,41847,1731143603026/9b029db41fd9%2C41847%2C1731143603026.1731143604973 2024-11-09T09:13:25,042 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36227:36227)] 2024-11-09T09:13:25,043 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:13:25,043 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:25,046 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,047 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T09:13:25,104 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:25,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:25,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T09:13:25,110 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:25,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:13:25,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T09:13:25,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:25,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:13:25,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T09:13:25,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:25,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:13:25,123 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,124 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,133 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T09:13:25,138 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T09:13:25,144 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:13:25,145 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65042997, jitterRate=-0.030783817172050476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T09:13:25,152 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T09:13:25,154 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T09:13:25,179 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6144866a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:13:25,208 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-09T09:13:25,218 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T09:13:25,218 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T09:13:25,220 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T09:13:25,221 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T09:13:25,226 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-11-09T09:13:25,226 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T09:13:25,248 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T09:13:25,260 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T09:13:25,300 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-09T09:13:25,303 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T09:13:25,304 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T09:13:25,315 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-09T09:13:25,318 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T09:13:25,323 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T09:13:25,333 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-09T09:13:25,334 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T09:13:25,341 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T09:13:25,354 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T09:13:25,357 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T09:13:25,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T09:13:25,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T09:13:25,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:25,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:25,367 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=9b029db41fd9,41847,1731143603026, sessionid=0x1012028b2df0000, setting cluster-up flag (Was=false) 2024-11-09T09:13:25,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:25,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:25,416 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T09:13:25,418 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9b029db41fd9,41847,1731143603026 2024-11-09T09:13:25,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:25,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:25,466 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T09:13:25,469 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9b029db41fd9,41847,1731143603026 2024-11-09T09:13:25,539 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9b029db41fd9:43449 2024-11-09T09:13:25,541 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1008): ClusterId : 713e361b-88e2-4321-a1ee-2409650bfc38 2024-11-09T09:13:25,543 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-09T09:13:25,544 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T09:13:25,548 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-09T09:13:25,550 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T09:13:25,551 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T09:13:25,552 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T09:13:25,555 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9b029db41fd9,41847,1731143603026 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T09:13:25,558 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9b029db41fd9:0, corePoolSize=5, maxPoolSize=5 2024-11-09T09:13:25,559 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9b029db41fd9:0, corePoolSize=5, maxPoolSize=5 2024-11-09T09:13:25,559 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9b029db41fd9:0, corePoolSize=5, maxPoolSize=5 2024-11-09T09:13:25,559 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9b029db41fd9:0, corePoolSize=5, maxPoolSize=5 2024-11-09T09:13:25,559 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T09:13:25,559 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9b029db41fd9:0, corePoolSize=10, maxPoolSize=10 2024-11-09T09:13:25,559 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,559 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9b029db41fd9:0, corePoolSize=2, maxPoolSize=2 2024-11-09T09:13:25,560 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,560 DEBUG [RS:0;9b029db41fd9:43449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d67a89a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:13:25,561 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731143635561 2024-11-09T09:13:25,561 DEBUG [RS:0;9b029db41fd9:43449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b27f2a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9b029db41fd9/172.17.0.2:0 2024-11-09T09:13:25,562 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T09:13:25,563 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T09:13:25,564 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-09T09:13:25,564 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-09T09:13:25,564 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-09T09:13:25,565 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-09T09:13:25,565 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-09T09:13:25,566 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T09:13:25,567 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T09:13:25,567 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3073): reportForDuty to master=9b029db41fd9,41847,1731143603026 with isa=9b029db41fd9/172.17.0.2:43449, startcode=1731143603787 2024-11-09T09:13:25,567 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T09:13:25,567 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T09:13:25,569 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,570 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:25,570 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T09:13:25,570 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T09:13:25,572 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T09:13:25,572 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T09:13:25,575 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T09:13:25,576 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T09:13:25,577 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9b029db41fd9:0:becomeActiveMaster-HFileCleaner.large.0-1731143605577,5,FailOnTimeoutGroup] 2024-11-09T09:13:25,578 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9b029db41fd9:0:becomeActiveMaster-HFileCleaner.small.0-1731143605577,5,FailOnTimeoutGroup] 2024-11-09T09:13:25,579 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,579 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T09:13:25,580 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,580 DEBUG [RS:0;9b029db41fd9:43449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T09:13:25,580 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741831_1007 (size=1039) 2024-11-09T09:13:25,608 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46845, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T09:13:25,613 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 9b029db41fd9,43449,1731143603787 2024-11-09T09:13:25,615 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41847 {}] master.ServerManager(486): Registering regionserver=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:25,629 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:13:25,629 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:39925 2024-11-09T09:13:25,629 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-09T09:13:25,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T09:13:25,642 DEBUG [RS:0;9b029db41fd9:43449 {}] zookeeper.ZKUtil(111): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9b029db41fd9,43449,1731143603787 2024-11-09T09:13:25,642 WARN [RS:0;9b029db41fd9:43449 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T09:13:25,642 INFO [RS:0;9b029db41fd9:43449 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T09:13:25,642 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787 2024-11-09T09:13:25,645 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9b029db41fd9,43449,1731143603787] 2024-11-09T09:13:25,656 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-09T09:13:25,667 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T09:13:25,683 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T09:13:25,686 INFO [RS:0;9b029db41fd9:43449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T09:13:25,686 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,687 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-09T09:13:25,694 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9b029db41fd9:0, corePoolSize=2, maxPoolSize=2 2024-11-09T09:13:25,695 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,696 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,696 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,696 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,696 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9b029db41fd9:0, corePoolSize=1, maxPoolSize=1 2024-11-09T09:13:25,696 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9b029db41fd9:0, corePoolSize=3, maxPoolSize=3 2024-11-09T09:13:25,696 DEBUG [RS:0;9b029db41fd9:43449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9b029db41fd9:0, corePoolSize=3, maxPoolSize=3 2024-11-09T09:13:25,698 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,698 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,698 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,698 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,698 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,43449,1731143603787-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T09:13:25,719 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T09:13:25,721 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,43449,1731143603787-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:25,741 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.Replication(204): 9b029db41fd9,43449,1731143603787 started 2024-11-09T09:13:25,741 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1767): Serving as 9b029db41fd9,43449,1731143603787, RpcServer on 9b029db41fd9/172.17.0.2:43449, sessionid=0x1012028b2df0001 2024-11-09T09:13:25,742 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T09:13:25,742 DEBUG [RS:0;9b029db41fd9:43449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9b029db41fd9,43449,1731143603787 2024-11-09T09:13:25,742 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9b029db41fd9,43449,1731143603787' 2024-11-09T09:13:25,742 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T09:13:25,743 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T09:13:25,744 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T09:13:25,744 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T09:13:25,744 DEBUG [RS:0;9b029db41fd9:43449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9b029db41fd9,43449,1731143603787 2024-11-09T09:13:25,744 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9b029db41fd9,43449,1731143603787' 2024-11-09T09:13:25,744 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T09:13:25,745 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T09:13:25,745 DEBUG [RS:0;9b029db41fd9:43449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T09:13:25,746 INFO [RS:0;9b029db41fd9:43449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T09:13:25,746 INFO [RS:0;9b029db41fd9:43449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T09:13:25,986 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-09T09:13:25,986 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:13:26,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741832_1008 (size=32) 2024-11-09T09:13:26,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:26,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T09:13:26,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T09:13:26,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:26,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:26,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T09:13:26,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T09:13:26,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:26,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:26,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T09:13:26,017 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T09:13:26,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:26,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:26,021 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740 2024-11-09T09:13:26,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740 2024-11-09T09:13:26,025 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-09T09:13:26,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-09T09:13:26,032 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:13:26,033 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60656904, jitterRate=-0.09614169597625732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-09T09:13:26,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-09T09:13:26,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-09T09:13:26,036 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-09T09:13:26,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-09T09:13:26,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T09:13:26,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T09:13:26,038 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-09T09:13:26,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-09T09:13:26,041 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-09T09:13:26,041 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-09T09:13:26,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T09:13:26,054 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T09:13:26,056 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T09:13:26,207 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:27,208 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:28,209 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:29,210 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:30,212 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:31,213 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:32,214 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:33,215 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T09:13:33,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T09:13:34,216 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:35,217 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:36,218 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:37,219 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:38,221 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:39,222 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:40,224 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:41,225 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:42,226 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:43,227 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:44,228 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:45,229 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:46,230 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:47,231 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:48,233 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:49,234 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:50,235 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:51,236 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:52,224 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:13:52,237 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:52,399 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:13:53,238 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:54,239 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:55,240 WARN [9b029db41fd9:41847 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T09:13:55,759 INFO [RS:0;9b029db41fd9:43449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T09:13:55,763 INFO [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9b029db41fd9%2C43449%2C1731143603787, suffix=, logDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787, archiveDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/oldWALs, maxLogs=32 2024-11-09T09:13:55,778 DEBUG [RS:0;9b029db41fd9:43449 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787/9b029db41fd9%2C43449%2C1731143603787.1731143635765, exclude list is [], retry=0 2024-11-09T09:13:55,784 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34159,DS-976a2484-d0ae-4ecb-b005-d5dc8e3bc7b2,DISK] 2024-11-09T09:13:55,787 INFO [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787/9b029db41fd9%2C43449%2C1731143603787.1731143635765 2024-11-09T09:13:55,787 DEBUG [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36227:36227)] 2024-11-09T09:13:55,994 DEBUG [9b029db41fd9:41847 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T09:13:56,005 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:56,009 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9b029db41fd9,43449,1731143603787, state=OPENING 2024-11-09T09:13:56,191 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T09:13:56,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:56,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:56,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T09:13:56,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T09:13:56,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:13:56,383 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:13:56,385 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T09:13:56,388 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39246, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T09:13:56,398 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-09T09:13:56,398 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T09:13:56,399 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T09:13:56,402 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9b029db41fd9%2C43449%2C1731143603787.meta, suffix=.meta, logDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787, archiveDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/oldWALs, maxLogs=32 2024-11-09T09:13:56,416 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787/9b029db41fd9%2C43449%2C1731143603787.meta.1731143636404.meta, exclude list is [], retry=0 2024-11-09T09:13:56,420 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34159,DS-976a2484-d0ae-4ecb-b005-d5dc8e3bc7b2,DISK] 2024-11-09T09:13:56,422 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/WALs/9b029db41fd9,43449,1731143603787/9b029db41fd9%2C43449%2C1731143603787.meta.1731143636404.meta 2024-11-09T09:13:56,423 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36227:36227)] 2024-11-09T09:13:56,423 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:13:56,424 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T09:13:56,473 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T09:13:56,477 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T09:13:56,480 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T09:13:56,480 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:56,480 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-09T09:13:56,480 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-09T09:13:56,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T09:13:56,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T09:13:56,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:56,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:56,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T09:13:56,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T09:13:56,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:56,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:56,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T09:13:56,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T09:13:56,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:56,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T09:13:56,492 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740 2024-11-09T09:13:56,494 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740 2024-11-09T09:13:56,497 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-09T09:13:56,499 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-09T09:13:56,501 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63794124, jitterRate=-0.04939347505569458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-09T09:13:56,502 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-09T09:13:56,508 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731143636379 2024-11-09T09:13:56,517 DEBUG [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T09:13:56,518 INFO [RS_OPEN_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-09T09:13:56,519 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:56,521 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9b029db41fd9,43449,1731143603787, state=OPEN 2024-11-09T09:13:56,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T09:13:56,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T09:13:56,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T09:13:56,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T09:13:56,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T09:13:56,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=9b029db41fd9,43449,1731143603787 in 376 msec 2024-11-09T09:13:56,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T09:13:56,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.5430 sec 2024-11-09T09:13:56,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 31.0930 sec 2024-11-09T09:13:56,599 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731143636599, completionTime=-1 2024-11-09T09:13:56,599 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T09:13:56,600 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-09T09:13:56,631 DEBUG [hconnection-0x1d977fe5-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:13:56,634 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39258, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:13:56,644 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-09T09:13:56,645 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731143696645 2024-11-09T09:13:56,645 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731143756645 2024-11-09T09:13:56,645 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 45 msec 2024-11-09T09:13:56,676 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,41847,1731143603026-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:56,676 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,41847,1731143603026-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:56,676 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,41847,1731143603026-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:56,678 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9b029db41fd9:41847, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:56,678 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:56,684 DEBUG [master/9b029db41fd9:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-09T09:13:56,686 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-09T09:13:56,687 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T09:13:56,692 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-09T09:13:56,695 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:13:56,696 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:56,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:13:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741835_1011 (size=358) 2024-11-09T09:13:57,119 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => cc68045eb729be737b2bb40a8dcab3f4, NAME => 'hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:13:57,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741836_1012 (size=42) 2024-11-09T09:13:57,537 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:57,537 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing cc68045eb729be737b2bb40a8dcab3f4, disabling compactions & flushes 2024-11-09T09:13:57,538 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,538 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,538 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. after waiting 0 ms 2024-11-09T09:13:57,539 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,539 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,539 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for cc68045eb729be737b2bb40a8dcab3f4: 2024-11-09T09:13:57,541 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:13:57,549 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731143637543"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143637543"}]},"ts":"1731143637543"} 2024-11-09T09:13:57,562 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T09:13:57,565 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54566, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T09:13:57,572 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:13:57,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:13:57,577 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143637574"}]},"ts":"1731143637574"} 2024-11-09T09:13:57,581 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-09T09:13:57,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cc68045eb729be737b2bb40a8dcab3f4, ASSIGN}] 2024-11-09T09:13:57,632 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cc68045eb729be737b2bb40a8dcab3f4, ASSIGN 2024-11-09T09:13:57,636 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=cc68045eb729be737b2bb40a8dcab3f4, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:13:57,788 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cc68045eb729be737b2bb40a8dcab3f4, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:57,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure cc68045eb729be737b2bb40a8dcab3f4, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:13:57,953 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:13:57,966 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => cc68045eb729be737b2bb40a8dcab3f4, NAME => 'hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:13:57,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:57,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,969 INFO [StoreOpener-cc68045eb729be737b2bb40a8dcab3f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,972 INFO [StoreOpener-cc68045eb729be737b2bb40a8dcab3f4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc68045eb729be737b2bb40a8dcab3f4 columnFamilyName info 2024-11-09T09:13:57,972 DEBUG [StoreOpener-cc68045eb729be737b2bb40a8dcab3f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:57,973 INFO [StoreOpener-cc68045eb729be737b2bb40a8dcab3f4-1 {}] regionserver.HStore(327): Store=cc68045eb729be737b2bb40a8dcab3f4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:13:57,974 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,979 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:13:57,983 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:13:57,984 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened cc68045eb729be737b2bb40a8dcab3f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65219739, jitterRate=-0.028150156140327454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:13:57,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for cc68045eb729be737b2bb40a8dcab3f4: 2024-11-09T09:13:57,987 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4., pid=6, masterSystemTime=1731143637952 2024-11-09T09:13:57,990 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,991 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:13:57,991 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cc68045eb729be737b2bb40a8dcab3f4, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:57,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T09:13:58,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure cc68045eb729be737b2bb40a8dcab3f4, server=9b029db41fd9,43449,1731143603787 in 199 msec 2024-11-09T09:13:58,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T09:13:58,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=cc68045eb729be737b2bb40a8dcab3f4, ASSIGN in 372 msec 2024-11-09T09:13:58,005 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:13:58,005 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143638005"}]},"ts":"1731143638005"} 2024-11-09T09:13:58,008 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-09T09:13:58,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:13:58,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3570 sec 2024-11-09T09:13:58,098 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-09T09:13:58,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-09T09:13:58,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:58,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:13:58,141 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-09T09:13:58,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-09T09:13:58,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 40 msec 2024-11-09T09:13:58,186 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-09T09:13:58,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-09T09:13:58,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 33 msec 2024-11-09T09:13:58,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-09T09:13:58,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-09T09:13:58,266 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 34.399sec 2024-11-09T09:13:58,269 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T09:13:58,272 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T09:13:58,274 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T09:13:58,275 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T09:13:58,275 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T09:13:58,276 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,41847,1731143603026-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T09:13:58,276 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,41847,1731143603026-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T09:13:58,282 DEBUG [master/9b029db41fd9:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-09T09:13:58,282 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T09:13:58,283 INFO [master/9b029db41fd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9b029db41fd9,41847,1731143603026-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T09:13:58,286 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x799a723d to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5899569f 2024-11-09T09:13:58,286 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-09T09:13:58,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9982629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:13:58,303 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T09:13:58,303 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T09:13:58,311 DEBUG [hconnection-0x254a0222-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:13:58,335 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:13:58,344 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=9b029db41fd9,41847,1731143603026 2024-11-09T09:13:58,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=77, ProcessCount=11, AvailableMemoryMB=3796 2024-11-09T09:13:58,366 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6826b5e3 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8aedc26 2024-11-09T09:13:58,375 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x6826b5e3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3026e913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:13:58,379 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0ecca9d4 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70fd1bc7 2024-11-09T09:13:58,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5692e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:13:58,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T09:13:58,402 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52732, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T09:13:58,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:13:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-09T09:13:58,415 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:13:58,416 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:58,418 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:13:58,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-09T09:13:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T09:13:58,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741837_1013 (size=437) 2024-11-09T09:13:58,836 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 459f96eb1d922bcd9f696d43a6691e8c, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:13:58,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741838_1014 (size=88) 2024-11-09T09:13:59,254 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:59,254 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 459f96eb1d922bcd9f696d43a6691e8c, disabling compactions & flushes 2024-11-09T09:13:59,254 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. after waiting 0 ms 2024-11-09T09:13:59,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,255 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 459f96eb1d922bcd9f696d43a6691e8c: 2024-11-09T09:13:59,258 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:13:59,258 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731143639258"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143639258"}]},"ts":"1731143639258"} 2024-11-09T09:13:59,262 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:13:59,264 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:13:59,265 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143639264"}]},"ts":"1731143639264"} 2024-11-09T09:13:59,267 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:13:59,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=459f96eb1d922bcd9f696d43a6691e8c, ASSIGN}] 2024-11-09T09:13:59,311 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=459f96eb1d922bcd9f696d43a6691e8c, ASSIGN 2024-11-09T09:13:59,314 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=459f96eb1d922bcd9f696d43a6691e8c, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:13:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T09:13:59,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=459f96eb1d922bcd9f696d43a6691e8c, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:59,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 459f96eb1d922bcd9f696d43a6691e8c, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:13:59,630 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:13:59,643 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,643 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 459f96eb1d922bcd9f696d43a6691e8c, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:13:59,644 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,644 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:13:59,644 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,645 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,647 INFO [StoreOpener-459f96eb1d922bcd9f696d43a6691e8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,650 INFO [StoreOpener-459f96eb1d922bcd9f696d43a6691e8c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 459f96eb1d922bcd9f696d43a6691e8c columnFamilyName testFamily 2024-11-09T09:13:59,650 DEBUG [StoreOpener-459f96eb1d922bcd9f696d43a6691e8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:13:59,651 INFO [StoreOpener-459f96eb1d922bcd9f696d43a6691e8c-1 {}] regionserver.HStore(327): Store=459f96eb1d922bcd9f696d43a6691e8c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:13:59,653 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,653 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,657 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:13:59,661 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:13:59,662 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 459f96eb1d922bcd9f696d43a6691e8c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69058844, jitterRate=0.029056966304779053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:13:59,663 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 459f96eb1d922bcd9f696d43a6691e8c: 2024-11-09T09:13:59,664 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c., pid=11, masterSystemTime=1731143639630 2024-11-09T09:13:59,667 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,667 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:13:59,668 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=459f96eb1d922bcd9f696d43a6691e8c, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:13:59,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-09T09:13:59,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 459f96eb1d922bcd9f696d43a6691e8c, server=9b029db41fd9,43449,1731143603787 in 198 msec 2024-11-09T09:13:59,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T09:13:59,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=459f96eb1d922bcd9f696d43a6691e8c, ASSIGN in 368 msec 2024-11-09T09:13:59,680 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:13:59,680 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143639680"}]},"ts":"1731143639680"} 2024-11-09T09:13:59,683 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:13:59,692 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:13:59,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 1.2800 sec 2024-11-09T09:14:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T09:14:01,440 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-09T09:14:01,441 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:01,442 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:01,448 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:01,449 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:01,449 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-09T09:14:01,452 DEBUG [hconnection-0x4cef3c8b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:01,454 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53664, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:01,467 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:01,472 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T09:14:01,482 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:14:01,484 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x6826b5e3 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:14:01,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:01,490 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53666, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:01,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:14:01,500 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T09:14:01,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549835 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:01,503 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T09:14:01,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549835 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:01,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001781 2024-11-09T09:14:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53666 deadline: 1731143644502, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:01,524 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549835 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:01,527 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T09:14:01,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549836 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:01,528 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T09:14:01,529 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:01,530 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:01,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549837 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:01,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001779 2024-11-09T09:14:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53666 deadline: 1731143644530, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:01,532 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549837 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:01,534 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:01,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549838 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:01,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5312316530116001778 2024-11-09T09:14:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53666 deadline: 1731143644533, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:01,537 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:01,536 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549838 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:01,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549839 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:01,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5312316530116001777 2024-11-09T09:14:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53666 deadline: 1731143644536, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:01,540 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549839 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:01,542 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:01,542 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0ecca9d4 to 127.0.0.1:49767 2024-11-09T09:14:01,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:01,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:01,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:01,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:01,548 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6826b5e3 to 127.0.0.1:49767 2024-11-09T09:14:01,557 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=225 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-319759215_22 at /127.0.0.1:44180 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49767@0x6826b5e3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49767@0x6826b5e3-SendThread(127.0.0.1:49767) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49767@0x0ecca9d4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49767@0x0ecca9d4-SendThread(127.0.0.1:49767) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49767@0x6826b5e3 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$738/0x00007f0ec4b43fc0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49767@0x0ecca9d4 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$738/0x00007f0ec4b43fc0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x254a0222-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=77 (was 77), ProcessCount=11 (was 11), AvailableMemoryMB=3783 (was 3796) 2024-11-09T09:14:01,565 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=225, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=77, ProcessCount=11, AvailableMemoryMB=3782 2024-11-09T09:14:01,567 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6aced667 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3bf02dc4 2024-11-09T09:14:01,584 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x6aced667 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a234355, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:01,586 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0dc02d58 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3531ab68 2024-11-09T09:14:01,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd2ed6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:01,611 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:01,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-09T09:14:01,614 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:01,614 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:01,614 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-09T09:14:01,616 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T09:14:01,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741839_1015 (size=423) 2024-11-09T09:14:02,021 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:02,029 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ca546e27319aa177954e402fd053bd2b, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:02,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741840_1016 (size=74) 2024-11-09T09:14:02,044 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:02,044 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ca546e27319aa177954e402fd053bd2b, disabling compactions & flushes 2024-11-09T09:14:02,044 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,044 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,044 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. after waiting 0 ms 2024-11-09T09:14:02,044 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,044 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,045 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ca546e27319aa177954e402fd053bd2b: 2024-11-09T09:14:02,047 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:02,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731143642047"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143642047"}]},"ts":"1731143642047"} 2024-11-09T09:14:02,050 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:02,052 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:02,052 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143642052"}]},"ts":"1731143642052"} 2024-11-09T09:14:02,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-09T09:14:02,055 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:02,056 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T09:14:02,056 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-09T09:14:02,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ca546e27319aa177954e402fd053bd2b, ASSIGN}] 2024-11-09T09:14:02,101 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ca546e27319aa177954e402fd053bd2b, ASSIGN 2024-11-09T09:14:02,103 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ca546e27319aa177954e402fd053bd2b, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:02,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=ca546e27319aa177954e402fd053bd2b, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:02,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure ca546e27319aa177954e402fd053bd2b, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:02,409 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:02,414 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,414 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => ca546e27319aa177954e402fd053bd2b, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:02,414 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:02,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,417 INFO [StoreOpener-ca546e27319aa177954e402fd053bd2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,420 INFO [StoreOpener-ca546e27319aa177954e402fd053bd2b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca546e27319aa177954e402fd053bd2b columnFamilyName testFamily 2024-11-09T09:14:02,420 DEBUG [StoreOpener-ca546e27319aa177954e402fd053bd2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:02,421 INFO [StoreOpener-ca546e27319aa177954e402fd053bd2b-1 {}] regionserver.HStore(327): Store=ca546e27319aa177954e402fd053bd2b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:02,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,427 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for ca546e27319aa177954e402fd053bd2b 2024-11-09T09:14:02,432 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:02,433 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened ca546e27319aa177954e402fd053bd2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66416379, jitterRate=-0.010318830609321594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:02,434 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for ca546e27319aa177954e402fd053bd2b: 2024-11-09T09:14:02,436 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b., pid=14, masterSystemTime=1731143642409 2024-11-09T09:14:02,439 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,439 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:14:02,440 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=ca546e27319aa177954e402fd053bd2b, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:02,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T09:14:02,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure ca546e27319aa177954e402fd053bd2b, server=9b029db41fd9,43449,1731143603787 in 188 msec 2024-11-09T09:14:02,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-09T09:14:02,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ca546e27319aa177954e402fd053bd2b, ASSIGN in 349 msec 2024-11-09T09:14:02,451 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:02,452 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143642451"}]},"ts":"1731143642451"} 2024-11-09T09:14:02,454 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:02,493 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:02,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 882 msec 2024-11-09T09:14:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T09:14:02,623 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-09T09:14:02,623 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:02,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:02,630 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:02,630 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:02,631 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-09T09:14:02,633 DEBUG [hconnection-0x313cdc31-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:02,635 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53672, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:02,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:02,653 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:14:02,655 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:14:03,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-09T09:14:03,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:03,450 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T09:14:03,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:03,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-09T09:14:03,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:03,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T09:14:03,455 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T09:14:03,455 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-09T09:14:03,455 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:07,922 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:08,058 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-09T09:14:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T09:14:09,688 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:14:11,663 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11016: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11016: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:14:11,665 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T09:14:11.664Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11016: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11016: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1975ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:14:11,667 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9013 2024-11-09T09:14:11,668 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0dc02d58 to 127.0.0.1:49767 2024-11-09T09:14:11,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:11,668 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:11,668 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:11,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:11,669 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6aced667 to 127.0.0.1:49767 2024-11-09T09:14:11,682 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=225 (was 225), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=66 (was 77), ProcessCount=11 (was 11), AvailableMemoryMB=3770 (was 3782) 2024-11-09T09:14:11,692 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=66, ProcessCount=11, AvailableMemoryMB=3770 2024-11-09T09:14:11,695 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01b4a855 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b52d018 2024-11-09T09:14:11,760 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x01b4a855 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47806c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:11,765 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5afde8a1 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f7a6a21 2024-11-09T09:14:11,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b28819b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:11,777 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:11,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-09T09:14:11,780 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:11,781 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:11,781 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-09T09:14:11,782 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-09T09:14:11,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741841_1017 (size=421) 2024-11-09T09:14:12,195 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f90ee672bf70ce14e71d8eba9060419a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:12,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741842_1018 (size=72) 2024-11-09T09:14:12,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:12,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing f90ee672bf70ce14e71d8eba9060419a, disabling compactions & flushes 2024-11-09T09:14:12,610 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:12,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:12,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. after waiting 0 ms 2024-11-09T09:14:12,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:12,611 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:12,611 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for f90ee672bf70ce14e71d8eba9060419a: 2024-11-09T09:14:12,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:12,617 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731143652616"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143652616"}]},"ts":"1731143652616"} 2024-11-09T09:14:12,622 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:12,624 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:12,624 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143652624"}]},"ts":"1731143652624"} 2024-11-09T09:14:12,627 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:12,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f90ee672bf70ce14e71d8eba9060419a, ASSIGN}] 2024-11-09T09:14:12,677 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f90ee672bf70ce14e71d8eba9060419a, ASSIGN 2024-11-09T09:14:12,680 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f90ee672bf70ce14e71d8eba9060419a, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-09T09:14:12,831 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=f90ee672bf70ce14e71d8eba9060419a, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:12,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure f90ee672bf70ce14e71d8eba9060419a, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:12,993 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:13,005 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:13,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => f90ee672bf70ce14e71d8eba9060419a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:13,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:13,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,009 INFO [StoreOpener-f90ee672bf70ce14e71d8eba9060419a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,012 INFO [StoreOpener-f90ee672bf70ce14e71d8eba9060419a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f90ee672bf70ce14e71d8eba9060419a columnFamilyName testFamily 2024-11-09T09:14:13,012 DEBUG [StoreOpener-f90ee672bf70ce14e71d8eba9060419a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:13,013 INFO [StoreOpener-f90ee672bf70ce14e71d8eba9060419a-1 {}] regionserver.HStore(327): Store=f90ee672bf70ce14e71d8eba9060419a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:13,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,018 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:14:13,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:13,022 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened f90ee672bf70ce14e71d8eba9060419a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66789377, jitterRate=-0.004760727286338806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:13,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for f90ee672bf70ce14e71d8eba9060419a: 2024-11-09T09:14:13,024 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a., pid=17, masterSystemTime=1731143652993 2024-11-09T09:14:13,027 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:13,027 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:14:13,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=f90ee672bf70ce14e71d8eba9060419a, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:13,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-09T09:14:13,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure f90ee672bf70ce14e71d8eba9060419a, server=9b029db41fd9,43449,1731143603787 in 191 msec 2024-11-09T09:14:13,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T09:14:13,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f90ee672bf70ce14e71d8eba9060419a, ASSIGN in 359 msec 2024-11-09T09:14:13,037 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:13,037 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143653037"}]},"ts":"1731143653037"} 2024-11-09T09:14:13,039 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:13,076 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:13,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 1.2990 sec 2024-11-09T09:14:13,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-09T09:14:13,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:14,061 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-09T09:14:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-09T09:14:14,793 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-09T09:14:14,793 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:14,794 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:14,804 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:14,804 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:14,805 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-09T09:14:14,808 DEBUG [hconnection-0x60663d7e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:14,810 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55774, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:14,846 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:14,851 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:14:14,851 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:14:14,853 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x01b4a855 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:14:14,854 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x01b4a855 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:14,856 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55788, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:14:14,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:14:14,862 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549851 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:14,863 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:14:14,864 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549851 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:14,865 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-09T09:14:16,190 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:14:18,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:22,224 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:14:23,873 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9009 2024-11-09T09:14:23,874 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5afde8a1 to 127.0.0.1:49767 2024-11-09T09:14:23,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:23,875 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:23,874 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549851 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:23,876 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:23,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:23,876 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01b4a855 to 127.0.0.1:49767 2024-11-09T09:14:23,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001765' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T09:14:23,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:55788 deadline: 1731143669873, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001765' does not exist 2024-11-09T09:14:23,883 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:14:23,877 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9b029db41fd9,43449,1731143603787 for closing scanner id = -5312316530116001765 for f90ee672bf70ce14e71d8eba9060419a of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T09:14:23,889 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=218 (was 225), OpenFileDescriptor=448 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=56 (was 66), ProcessCount=11 (was 11), AvailableMemoryMB=3758 (was 3770) 2024-11-09T09:14:23,897 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=218, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=56, ProcessCount=11, AvailableMemoryMB=3758 2024-11-09T09:14:23,900 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x198cc5db to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30ebee64 2024-11-09T09:14:23,934 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x198cc5db {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5f94f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:23,938 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71f012ce to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65a7f185 2024-11-09T09:14:23,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7a52f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:23,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-09T09:14:23,959 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:23,959 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:23,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-09T09:14:23,960 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T09:14:23,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741843_1019 (size=432) 2024-11-09T09:14:24,366 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:14:24,376 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f6ef5e815e8a0538da916d41f9c6bba5, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:24,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741844_1020 (size=83) 2024-11-09T09:14:24,791 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:24,791 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing f6ef5e815e8a0538da916d41f9c6bba5, disabling compactions & flushes 2024-11-09T09:14:24,792 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:24,792 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:24,792 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. after waiting 0 ms 2024-11-09T09:14:24,792 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:24,792 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:24,792 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for f6ef5e815e8a0538da916d41f9c6bba5: 2024-11-09T09:14:24,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:24,795 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731143664795"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143664795"}]},"ts":"1731143664795"} 2024-11-09T09:14:24,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:24,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:24,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143664800"}]},"ts":"1731143664800"} 2024-11-09T09:14:24,802 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:24,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=f6ef5e815e8a0538da916d41f9c6bba5, ASSIGN}] 2024-11-09T09:14:24,876 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=f6ef5e815e8a0538da916d41f9c6bba5, ASSIGN 2024-11-09T09:14:24,877 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=f6ef5e815e8a0538da916d41f9c6bba5, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T09:14:25,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=f6ef5e815e8a0538da916d41f9c6bba5, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:25,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure f6ef5e815e8a0538da916d41f9c6bba5, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:25,189 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:25,202 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:25,202 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => f6ef5e815e8a0538da916d41f9c6bba5, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:25,203 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,203 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:25,204 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,204 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,208 INFO [StoreOpener-f6ef5e815e8a0538da916d41f9c6bba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,211 INFO [StoreOpener-f6ef5e815e8a0538da916d41f9c6bba5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6ef5e815e8a0538da916d41f9c6bba5 columnFamilyName testFamily 2024-11-09T09:14:25,211 DEBUG [StoreOpener-f6ef5e815e8a0538da916d41f9c6bba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:25,212 INFO [StoreOpener-f6ef5e815e8a0538da916d41f9c6bba5-1 {}] regionserver.HStore(327): Store=f6ef5e815e8a0538da916d41f9c6bba5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:25,215 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,215 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,218 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:14:25,221 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:25,221 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened f6ef5e815e8a0538da916d41f9c6bba5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61750544, jitterRate=-0.07984519004821777}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:25,222 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for f6ef5e815e8a0538da916d41f9c6bba5: 2024-11-09T09:14:25,223 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5., pid=20, masterSystemTime=1731143665188 2024-11-09T09:14:25,225 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:25,225 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:14:25,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=f6ef5e815e8a0538da916d41f9c6bba5, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:25,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T09:14:25,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure f6ef5e815e8a0538da916d41f9c6bba5, server=9b029db41fd9,43449,1731143603787 in 193 msec 2024-11-09T09:14:25,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-09T09:14:25,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=f6ef5e815e8a0538da916d41f9c6bba5, ASSIGN in 358 msec 2024-11-09T09:14:25,235 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:25,235 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143665235"}]},"ts":"1731143665235"} 2024-11-09T09:14:25,237 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:25,248 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:25,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.2930 sec 2024-11-09T09:14:25,785 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:14:26,065 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-09T09:14:26,708 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:14:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T09:14:26,973 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-09T09:14:26,973 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:26,974 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:26,979 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:26,980 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:26,980 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-09T09:14:26,982 DEBUG [hconnection-0xb66c34c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:26,984 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:26,996 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:26,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T09:14:27,001 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T09:14:27,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549855 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:27,002 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T09:14:27,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549855 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:27,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001761 2024-11-09T09:14:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44398 deadline: 1731143668003, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:27,004 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:14:27,006 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T09:14:27,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549856 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:27,008 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T09:14:27,008 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:27,010 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:27,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549857 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:27,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001759 2024-11-09T09:14:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44398 deadline: 1731143668010, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:27,011 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:14:27,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:27,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549858 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:27,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5312316530116001758 2024-11-09T09:14:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44398 deadline: 1731143668014, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:27,015 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:14:27,017 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:27,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549859 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:27,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5312316530116001757 2024-11-09T09:14:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44398 deadline: 1731143668018, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:14:27,019 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:14:27,021 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:14:27,022 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x71f012ce to 127.0.0.1:49767 2024-11-09T09:14:27,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:27,022 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:27,022 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:27,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:27,022 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x198cc5db to 127.0.0.1:49767 2024-11-09T09:14:27,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=223 (was 218) - Thread LEAK? -, OpenFileDescriptor=454 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=51 (was 56), ProcessCount=11 (was 11), AvailableMemoryMB=3749 (was 3758) 2024-11-09T09:14:27,039 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=223, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=51, ProcessCount=11, AvailableMemoryMB=3749 2024-11-09T09:14:27,042 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57b00a98 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@476f4049 2024-11-09T09:14:27,073 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x57b00a98 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545259e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:27,076 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29a5b61a to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c60db78 2024-11-09T09:14:27,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b53e0a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:27,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-09T09:14:27,088 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:27,088 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:27,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-09T09:14:27,089 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-09T09:14:27,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741845_1021 (size=426) 2024-11-09T09:14:27,506 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 46c966491675fb74bc449a58cad47886, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:27,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741846_1022 (size=77) 2024-11-09T09:14:27,926 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:27,926 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 46c966491675fb74bc449a58cad47886, disabling compactions & flushes 2024-11-09T09:14:27,927 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:27,927 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:27,927 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. after waiting 0 ms 2024-11-09T09:14:27,927 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:27,927 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:27,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 46c966491675fb74bc449a58cad47886: 2024-11-09T09:14:27,932 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:27,933 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731143667932"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143667932"}]},"ts":"1731143667932"} 2024-11-09T09:14:27,937 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:27,938 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:27,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143667938"}]},"ts":"1731143667938"} 2024-11-09T09:14:27,940 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:27,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=46c966491675fb74bc449a58cad47886, ASSIGN}] 2024-11-09T09:14:27,992 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=46c966491675fb74bc449a58cad47886, ASSIGN 2024-11-09T09:14:27,994 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=46c966491675fb74bc449a58cad47886, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-09T09:14:28,145 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=46c966491675fb74bc449a58cad47886, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:28,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 46c966491675fb74bc449a58cad47886, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:28,308 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:28,318 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:28,319 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => 46c966491675fb74bc449a58cad47886, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:28,320 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,320 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:28,321 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for 46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,321 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for 46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,323 INFO [StoreOpener-46c966491675fb74bc449a58cad47886-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,326 INFO [StoreOpener-46c966491675fb74bc449a58cad47886-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46c966491675fb74bc449a58cad47886 columnFamilyName testFamily 2024-11-09T09:14:28,326 DEBUG [StoreOpener-46c966491675fb74bc449a58cad47886-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:28,327 INFO [StoreOpener-46c966491675fb74bc449a58cad47886-1 {}] regionserver.HStore(327): Store=46c966491675fb74bc449a58cad47886/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:28,328 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,328 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,331 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for 46c966491675fb74bc449a58cad47886 2024-11-09T09:14:28,333 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:28,334 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened 46c966491675fb74bc449a58cad47886; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73403479, jitterRate=0.09379707276821136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:28,335 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for 46c966491675fb74bc449a58cad47886: 2024-11-09T09:14:28,336 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886., pid=23, masterSystemTime=1731143668308 2024-11-09T09:14:28,338 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:28,338 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:14:28,338 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=46c966491675fb74bc449a58cad47886, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:28,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-09T09:14:28,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 46c966491675fb74bc449a58cad47886, server=9b029db41fd9,43449,1731143603787 in 188 msec 2024-11-09T09:14:28,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T09:14:28,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=46c966491675fb74bc449a58cad47886, ASSIGN in 352 msec 2024-11-09T09:14:28,346 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:28,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143668346"}]},"ts":"1731143668346"} 2024-11-09T09:14:28,348 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:28,390 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:28,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.3050 sec 2024-11-09T09:14:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-09T09:14:30,102 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-09T09:14:30,102 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:30,103 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:30,109 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:30,109 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:30,109 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-09T09:14:30,112 DEBUG [hconnection-0xcb1c266-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:30,114 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44088, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:30,146 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:30,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:14:30,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:14:30,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-09T09:14:30,730 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T09:14:32,068 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-09T09:14:32,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:14:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44088 deadline: 1731143673154, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:14:32,182 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:14:32,182 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549864 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:14:32,183 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2032 2024-11-09T09:14:32,184 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29a5b61a to 127.0.0.1:49767 2024-11-09T09:14:32,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:32,184 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:32,185 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:32,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:32,185 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57b00a98 to 127.0.0.1:49767 2024-11-09T09:14:32,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226 (was 223) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=47 (was 51), ProcessCount=11 (was 11), AvailableMemoryMB=3731 (was 3749) 2024-11-09T09:14:32,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=226, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=47, ProcessCount=11, AvailableMemoryMB=3731 2024-11-09T09:14:32,209 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2498b190 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@753cd753 2024-11-09T09:14:32,241 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x2498b190 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f8944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:32,246 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20228930 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28fac797 2024-11-09T09:14:32,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e455e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:32,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-09T09:14:32,262 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:32,263 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:32,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-09T09:14:32,264 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T09:14:32,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741847_1023 (size=427) 2024-11-09T09:14:32,679 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 496f9744c3b0aceac24ffc23bd44774f, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741848_1024 (size=78) 2024-11-09T09:14:33,093 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:33,094 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 496f9744c3b0aceac24ffc23bd44774f, disabling compactions & flushes 2024-11-09T09:14:33,094 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,094 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,094 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. after waiting 0 ms 2024-11-09T09:14:33,094 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,095 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,095 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 496f9744c3b0aceac24ffc23bd44774f: 2024-11-09T09:14:33,099 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:33,099 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731143673099"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143673099"}]},"ts":"1731143673099"} 2024-11-09T09:14:33,103 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:33,104 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:33,104 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143673104"}]},"ts":"1731143673104"} 2024-11-09T09:14:33,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:33,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=496f9744c3b0aceac24ffc23bd44774f, ASSIGN}] 2024-11-09T09:14:33,160 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=496f9744c3b0aceac24ffc23bd44774f, ASSIGN 2024-11-09T09:14:33,163 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=496f9744c3b0aceac24ffc23bd44774f, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T09:14:33,314 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=496f9744c3b0aceac24ffc23bd44774f, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:33,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 496f9744c3b0aceac24ffc23bd44774f, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-09T09:14:33,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:33,448 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-09T09:14:33,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:33,476 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:33,488 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,489 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => 496f9744c3b0aceac24ffc23bd44774f, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:33,489 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,489 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:33,490 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for 496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,490 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for 496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,492 INFO [StoreOpener-496f9744c3b0aceac24ffc23bd44774f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,494 INFO [StoreOpener-496f9744c3b0aceac24ffc23bd44774f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 496f9744c3b0aceac24ffc23bd44774f columnFamilyName testFamily 2024-11-09T09:14:33,494 DEBUG [StoreOpener-496f9744c3b0aceac24ffc23bd44774f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:33,495 INFO [StoreOpener-496f9744c3b0aceac24ffc23bd44774f-1 {}] regionserver.HStore(327): Store=496f9744c3b0aceac24ffc23bd44774f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:33,496 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,497 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,501 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for 496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:14:33,505 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:33,507 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened 496f9744c3b0aceac24ffc23bd44774f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66458210, jitterRate=-0.009695500135421753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:33,507 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for 496f9744c3b0aceac24ffc23bd44774f: 2024-11-09T09:14:33,508 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f., pid=26, masterSystemTime=1731143673476 2024-11-09T09:14:33,510 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,511 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:14:33,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=496f9744c3b0aceac24ffc23bd44774f, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:33,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T09:14:33,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 496f9744c3b0aceac24ffc23bd44774f, server=9b029db41fd9,43449,1731143603787 in 193 msec 2024-11-09T09:14:33,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-09T09:14:33,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=496f9744c3b0aceac24ffc23bd44774f, ASSIGN in 358 msec 2024-11-09T09:14:33,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:33,519 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143673519"}]},"ts":"1731143673519"} 2024-11-09T09:14:33,521 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:33,566 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:33,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 1.3060 sec 2024-11-09T09:14:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T09:14:35,275 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-09T09:14:35,276 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:35,277 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:35,285 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:35,285 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:35,285 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-09T09:14:35,287 DEBUG [hconnection-0x69a6ddf1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:35,289 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44098, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:35,335 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:35,340 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:14:35,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:14:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T09:14:37,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:14:38,073 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-09T09:14:38,353 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:14:38,354 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T09:14:38.354Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:14:38,356 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3015 2024-11-09T09:14:38,357 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20228930 to 127.0.0.1:49767 2024-11-09T09:14:38,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:38,358 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:38,358 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:38,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:38,359 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2498b190 to 127.0.0.1:49767 2024-11-09T09:14:38,368 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225 (was 226), OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=43 (was 47), ProcessCount=11 (was 11), AvailableMemoryMB=3733 (was 3731) - AvailableMemoryMB LEAK? - 2024-11-09T09:14:38,377 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=43, ProcessCount=11, AvailableMemoryMB=3732 2024-11-09T09:14:38,380 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54f84884 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57a8d6f6 2024-11-09T09:14:38,416 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x54f84884 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579c6a9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:38,419 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69737ce2 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@481ee654 2024-11-09T09:14:38,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e342bbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:38,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-09T09:14:38,442 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:38,442 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:38,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-09T09:14:38,444 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-09T09:14:38,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741849_1025 (size=425) 2024-11-09T09:14:38,485 INFO [master/9b029db41fd9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T09:14:38,485 INFO [master/9b029db41fd9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T09:14:38,858 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2ed876c86d7febba0ebb3221271b6b23, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:38,868 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:14:38,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741850_1026 (size=76) 2024-11-09T09:14:38,993 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:39,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:39,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2ed876c86d7febba0ebb3221271b6b23, disabling compactions & flushes 2024-11-09T09:14:39,274 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. after waiting 0 ms 2024-11-09T09:14:39,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,275 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2ed876c86d7febba0ebb3221271b6b23: 2024-11-09T09:14:39,279 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:39,280 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731143679279"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143679279"}]},"ts":"1731143679279"} 2024-11-09T09:14:39,284 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:39,285 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:39,285 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143679285"}]},"ts":"1731143679285"} 2024-11-09T09:14:39,287 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:39,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=2ed876c86d7febba0ebb3221271b6b23, ASSIGN}] 2024-11-09T09:14:39,344 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=2ed876c86d7febba0ebb3221271b6b23, ASSIGN 2024-11-09T09:14:39,347 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=2ed876c86d7febba0ebb3221271b6b23, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-09T09:14:39,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=2ed876c86d7febba0ebb3221271b6b23, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:39,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 2ed876c86d7febba0ebb3221271b6b23, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:39,659 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:39,671 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,671 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => 2ed876c86d7febba0ebb3221271b6b23, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:39,672 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,672 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:39,672 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for 2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,672 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for 2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,674 INFO [StoreOpener-2ed876c86d7febba0ebb3221271b6b23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,676 INFO [StoreOpener-2ed876c86d7febba0ebb3221271b6b23-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ed876c86d7febba0ebb3221271b6b23 columnFamilyName testFamily 2024-11-09T09:14:39,676 DEBUG [StoreOpener-2ed876c86d7febba0ebb3221271b6b23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:39,676 INFO [StoreOpener-2ed876c86d7febba0ebb3221271b6b23-1 {}] regionserver.HStore(327): Store=2ed876c86d7febba0ebb3221271b6b23/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:39,677 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,678 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,681 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for 2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:14:39,684 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:39,685 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened 2ed876c86d7febba0ebb3221271b6b23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71451868, jitterRate=0.06471580266952515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:39,685 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for 2ed876c86d7febba0ebb3221271b6b23: 2024-11-09T09:14:39,686 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23., pid=29, masterSystemTime=1731143679659 2024-11-09T09:14:39,688 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,688 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:14:39,689 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=2ed876c86d7febba0ebb3221271b6b23, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:39,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-09T09:14:39,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 2ed876c86d7febba0ebb3221271b6b23, server=9b029db41fd9,43449,1731143603787 in 186 msec 2024-11-09T09:14:39,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T09:14:39,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=2ed876c86d7febba0ebb3221271b6b23, ASSIGN in 352 msec 2024-11-09T09:14:39,697 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:39,697 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143679697"}]},"ts":"1731143679697"} 2024-11-09T09:14:39,699 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:39,708 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:39,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.2710 sec 2024-11-09T09:14:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-09T09:14:41,451 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-09T09:14:41,451 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:41,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:41,457 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:41,457 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:41,458 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-09T09:14:41,460 DEBUG [hconnection-0x7dd94f2c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:41,463 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48086, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:41,472 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:41,475 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:14:41,475 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:14:41,477 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x54f84884 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:14:41,477 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x54f84884 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:41,479 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48090, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:14:41,483 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:14:41,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549876 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:41,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:14:41,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549876 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:14:41,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-09T09:14:43,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-09T09:14:43,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:43,448 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-09T09:14:43,449 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:44,077 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-09T09:14:44,493 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3008 2024-11-09T09:14:44,493 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69737ce2 to 127.0.0.1:49767 2024-11-09T09:14:44,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:44,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:44,494 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549876 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:44,495 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:44,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:44,495 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x54f84884 to 127.0.0.1:49767 2024-11-09T09:14:44,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001740' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T09:14:44,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:48090 deadline: 1731143685493, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001740' does not exist 2024-11-09T09:14:44,501 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:14:44,496 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9b029db41fd9,43449,1731143603787 for closing scanner id = -5312316530116001740 for 2ed876c86d7febba0ebb3221271b6b23 of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T09:14:44,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=40 (was 43), ProcessCount=11 (was 11), AvailableMemoryMB=3726 (was 3732) 2024-11-09T09:14:44,519 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=40, ProcessCount=11, AvailableMemoryMB=3726 2024-11-09T09:14:44,522 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69f7e9ca to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5584b6d9 2024-11-09T09:14:44,548 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x69f7e9ca {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93fbd8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:44,551 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76109287 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a453bf 2024-11-09T09:14:44,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b5338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:44,568 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:44,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-09T09:14:44,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:44,573 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:44,573 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-09T09:14:44,574 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T09:14:44,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741851_1027 (size=417) 2024-11-09T09:14:44,986 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:14:44,992 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 00c12a1ffd145dac3402d64c48cc20c7, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741852_1028 (size=68) 2024-11-09T09:14:45,177 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:45,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:45,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 00c12a1ffd145dac3402d64c48cc20c7, disabling compactions & flushes 2024-11-09T09:14:45,404 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. after waiting 0 ms 2024-11-09T09:14:45,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,404 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,404 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 00c12a1ffd145dac3402d64c48cc20c7: 2024-11-09T09:14:45,406 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:45,406 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731143685406"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143685406"}]},"ts":"1731143685406"} 2024-11-09T09:14:45,409 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:45,410 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:45,411 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143685410"}]},"ts":"1731143685410"} 2024-11-09T09:14:45,413 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:45,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=00c12a1ffd145dac3402d64c48cc20c7, ASSIGN}] 2024-11-09T09:14:45,468 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=00c12a1ffd145dac3402d64c48cc20c7, ASSIGN 2024-11-09T09:14:45,470 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=00c12a1ffd145dac3402d64c48cc20c7, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T09:14:45,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=00c12a1ffd145dac3402d64c48cc20c7, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:45,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 00c12a1ffd145dac3402d64c48cc20c7, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:45,783 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:45,793 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,794 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 00c12a1ffd145dac3402d64c48cc20c7, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:45,794 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:45,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,797 INFO [StoreOpener-00c12a1ffd145dac3402d64c48cc20c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,800 INFO [StoreOpener-00c12a1ffd145dac3402d64c48cc20c7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c12a1ffd145dac3402d64c48cc20c7 columnFamilyName testFamily 2024-11-09T09:14:45,800 DEBUG [StoreOpener-00c12a1ffd145dac3402d64c48cc20c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:45,801 INFO [StoreOpener-00c12a1ffd145dac3402d64c48cc20c7-1 {}] regionserver.HStore(327): Store=00c12a1ffd145dac3402d64c48cc20c7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:45,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,807 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:14:45,809 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:45,810 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 00c12a1ffd145dac3402d64c48cc20c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70195002, jitterRate=0.04598703980445862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:45,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 00c12a1ffd145dac3402d64c48cc20c7: 2024-11-09T09:14:45,811 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7., pid=32, masterSystemTime=1731143685783 2024-11-09T09:14:45,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,814 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:14:45,814 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=00c12a1ffd145dac3402d64c48cc20c7, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:45,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T09:14:45,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 00c12a1ffd145dac3402d64c48cc20c7, server=9b029db41fd9,43449,1731143603787 in 189 msec 2024-11-09T09:14:45,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T09:14:45,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=00c12a1ffd145dac3402d64c48cc20c7, ASSIGN in 354 msec 2024-11-09T09:14:45,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:45,822 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143685822"}]},"ts":"1731143685822"} 2024-11-09T09:14:45,824 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:45,833 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:45,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 1.2650 sec 2024-11-09T09:14:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T09:14:47,585 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-09T09:14:47,585 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:47,586 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:47,594 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:47,594 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:47,595 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-09T09:14:47,597 DEBUG [hconnection-0x7d3af8b6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:47,598 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48092, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:47,606 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:47,610 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:14:47,612 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549880 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:47,612 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-09T09:14:50,081 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-09T09:14:51,299 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:14:52,225 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:14:53,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-09T09:14:53,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:14:54,114 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6503 2024-11-09T09:14:54,114 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76109287 to 127.0.0.1:49767 2024-11-09T09:14:54,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:54,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:54,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:54,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:54,115 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69f7e9ca to 127.0.0.1:49767 2024-11-09T09:14:54,130 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 227) - Thread LEAK? -, OpenFileDescriptor=452 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=34 (was 40), ProcessCount=11 (was 11), AvailableMemoryMB=3714 (was 3726) 2024-11-09T09:14:54,141 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=34, ProcessCount=11, AvailableMemoryMB=3714 2024-11-09T09:14:54,144 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56eea45f to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d97f1b0 2024-11-09T09:14:54,172 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x56eea45f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c6417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:54,173 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c689328 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1bff745e 2024-11-09T09:14:54,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3890f722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:54,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-09T09:14:54,187 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:54,188 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:54,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-09T09:14:54,189 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-09T09:14:54,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741853_1029 (size=421) 2024-11-09T09:14:54,606 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ba96f0895981853116801be5af09f3d2, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:54,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741854_1030 (size=72) 2024-11-09T09:14:55,018 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:55,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ba96f0895981853116801be5af09f3d2, disabling compactions & flushes 2024-11-09T09:14:55,019 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. after waiting 0 ms 2024-11-09T09:14:55,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,019 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ba96f0895981853116801be5af09f3d2: 2024-11-09T09:14:55,022 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:55,022 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731143695022"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143695022"}]},"ts":"1731143695022"} 2024-11-09T09:14:55,026 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:55,027 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:55,028 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143695028"}]},"ts":"1731143695028"} 2024-11-09T09:14:55,029 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:55,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=ba96f0895981853116801be5af09f3d2, ASSIGN}] 2024-11-09T09:14:55,101 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=ba96f0895981853116801be5af09f3d2, ASSIGN 2024-11-09T09:14:55,102 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=ba96f0895981853116801be5af09f3d2, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-09T09:14:55,253 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=ba96f0895981853116801be5af09f3d2, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:55,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure ba96f0895981853116801be5af09f3d2, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:14:55,413 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:14:55,424 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,425 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => ba96f0895981853116801be5af09f3d2, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:14:55,425 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,425 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:55,426 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,426 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,429 INFO [StoreOpener-ba96f0895981853116801be5af09f3d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,431 INFO [StoreOpener-ba96f0895981853116801be5af09f3d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba96f0895981853116801be5af09f3d2 columnFamilyName testFamily 2024-11-09T09:14:55,431 DEBUG [StoreOpener-ba96f0895981853116801be5af09f3d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:55,431 INFO [StoreOpener-ba96f0895981853116801be5af09f3d2-1 {}] regionserver.HStore(327): Store=ba96f0895981853116801be5af09f3d2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:14:55,433 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,433 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,436 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for ba96f0895981853116801be5af09f3d2 2024-11-09T09:14:55,439 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:14:55,439 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened ba96f0895981853116801be5af09f3d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63989446, jitterRate=-0.04648295044898987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:14:55,440 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for ba96f0895981853116801be5af09f3d2: 2024-11-09T09:14:55,441 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2., pid=35, masterSystemTime=1731143695412 2024-11-09T09:14:55,443 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,443 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:14:55,444 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=ba96f0895981853116801be5af09f3d2, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:55,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T09:14:55,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure ba96f0895981853116801be5af09f3d2, server=9b029db41fd9,43449,1731143603787 in 188 msec 2024-11-09T09:14:55,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T09:14:55,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=ba96f0895981853116801be5af09f3d2, ASSIGN in 349 msec 2024-11-09T09:14:55,451 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:14:55,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143695451"}]},"ts":"1731143695451"} 2024-11-09T09:14:55,453 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:14:55,497 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:14:55,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 1.3120 sec 2024-11-09T09:14:56,085 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-09T09:14:56,693 DEBUG [master/9b029db41fd9:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T09:14:56,703 DEBUG [master/9b029db41fd9:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region cc68045eb729be737b2bb40a8dcab3f4 changed from -1.0 to 0.0, refreshing cache 2024-11-09T09:14:57,098 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:14:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-09T09:14:57,197 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-09T09:14:57,197 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:14:57,198 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:57,209 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-09T09:14:57,209 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:14:57,209 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-09T09:14:57,212 DEBUG [hconnection-0x5deabb5d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:14:57,214 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34286, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:14:57,224 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:14:57,227 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:14:57,230 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549884 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:14:57,230 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-09T09:14:58,337 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:14:58,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1504 2024-11-09T09:14:58,733 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c689328 to 127.0.0.1:49767 2024-11-09T09:14:58,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:58,734 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:14:58,734 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:14:58,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:14:58,735 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x56eea45f to 127.0.0.1:49767 2024-11-09T09:14:58,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=229 (was 228) - Thread LEAK? -, OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=31 (was 34), ProcessCount=11 (was 11), AvailableMemoryMB=3709 (was 3714) 2024-11-09T09:14:58,758 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=229, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=31, ProcessCount=11, AvailableMemoryMB=3708 2024-11-09T09:14:58,761 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7402cd34 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a149e2 2024-11-09T09:14:58,781 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7402cd34 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7911d63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:58,784 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22db89dd to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@17d9acc8 2024-11-09T09:14:58,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d0a86a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:14:58,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:14:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-09T09:14:58,826 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:14:58,826 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:14:58,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-09T09:14:58,827 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T09:14:58,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741855_1031 (size=420) 2024-11-09T09:14:59,240 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b502209f5d68712d0c9e5af7a60429f6, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:14:59,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741856_1032 (size=71) 2024-11-09T09:14:59,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:14:59,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing b502209f5d68712d0c9e5af7a60429f6, disabling compactions & flushes 2024-11-09T09:14:59,652 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:14:59,653 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:14:59,653 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. after waiting 0 ms 2024-11-09T09:14:59,653 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:14:59,653 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:14:59,653 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for b502209f5d68712d0c9e5af7a60429f6: 2024-11-09T09:14:59,655 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:14:59,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731143699655"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143699655"}]},"ts":"1731143699655"} 2024-11-09T09:14:59,659 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:14:59,660 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:14:59,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143699660"}]},"ts":"1731143699660"} 2024-11-09T09:14:59,662 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:14:59,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=b502209f5d68712d0c9e5af7a60429f6, ASSIGN}] 2024-11-09T09:14:59,716 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=b502209f5d68712d0c9e5af7a60429f6, ASSIGN 2024-11-09T09:14:59,717 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=b502209f5d68712d0c9e5af7a60429f6, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T09:14:59,869 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=b502209f5d68712d0c9e5af7a60429f6, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:14:59,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure b502209f5d68712d0c9e5af7a60429f6, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:00,030 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:00,041 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:15:00,042 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => b502209f5d68712d0c9e5af7a60429f6, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:00,043 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,043 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:00,043 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,043 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,045 INFO [StoreOpener-b502209f5d68712d0c9e5af7a60429f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,047 INFO [StoreOpener-b502209f5d68712d0c9e5af7a60429f6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b502209f5d68712d0c9e5af7a60429f6 columnFamilyName testFamily 2024-11-09T09:15:00,047 DEBUG [StoreOpener-b502209f5d68712d0c9e5af7a60429f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:00,048 INFO [StoreOpener-b502209f5d68712d0c9e5af7a60429f6-1 {}] regionserver.HStore(327): Store=b502209f5d68712d0c9e5af7a60429f6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:00,049 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,050 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,053 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:15:00,056 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:00,056 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened b502209f5d68712d0c9e5af7a60429f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69561577, jitterRate=0.036548271775245667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:00,057 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for b502209f5d68712d0c9e5af7a60429f6: 2024-11-09T09:15:00,058 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6., pid=38, masterSystemTime=1731143700029 2024-11-09T09:15:00,060 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:15:00,060 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:15:00,060 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=b502209f5d68712d0c9e5af7a60429f6, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:00,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-09T09:15:00,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure b502209f5d68712d0c9e5af7a60429f6, server=9b029db41fd9,43449,1731143603787 in 187 msec 2024-11-09T09:15:00,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-09T09:15:00,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=b502209f5d68712d0c9e5af7a60429f6, ASSIGN in 351 msec 2024-11-09T09:15:00,066 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:00,067 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143700067"}]},"ts":"1731143700067"} 2024-11-09T09:15:00,068 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:15:00,083 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:00,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 1.2600 sec 2024-11-09T09:15:00,930 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:15:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T09:15:01,838 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-09T09:15:01,838 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:15:01,839 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:01,846 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-09T09:15:01,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:01,847 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-09T09:15:01,849 DEBUG [hconnection-0x2cdec92b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:01,851 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57906, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:01,860 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:01,864 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x589961b8 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ed4d48 2024-11-09T09:15:01,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6f7724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:01,905 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:15:01,907 DEBUG [hconnection-0x57e1c679-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:01,909 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57910, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:01,913 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:15:01,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549889 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:01,915 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:15:01,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549889 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:01,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-09T09:15:02,088 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-09T09:15:02,923 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T09:15:02.923Z, RpcRetryingCaller{globalStartTime=2024-11-09T09:15:01.915Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T09:15:02.923Z, RpcRetryingCaller{globalStartTime=2024-11-09T09:15:01.915Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:15:02,924 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1008 2024-11-09T09:15:02,925 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x589961b8 to 127.0.0.1:49767 2024-11-09T09:15:02,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:02,926 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22db89dd to 127.0.0.1:49767 2024-11-09T09:15:02,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:02,927 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:02,927 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:02,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:02,928 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7402cd34 to 127.0.0.1:49767 2024-11-09T09:15:02,939 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 229) - Thread LEAK? -, OpenFileDescriptor=456 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=36 (was 31) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3696 (was 3708) 2024-11-09T09:15:02,949 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=36, ProcessCount=11, AvailableMemoryMB=3696 2024-11-09T09:15:02,952 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20c33f6f to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@504d60c6 2024-11-09T09:15:02,972 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x20c33f6f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e33ee1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:02,975 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5620f8e5 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@655b462c 2024-11-09T09:15:02,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cda25af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:02,994 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:02,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-09T09:15:02,997 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:02,997 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:02,997 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-09T09:15:02,998 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-09T09:15:03,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741857_1033 (size=422) 2024-11-09T09:15:03,416 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 271a1f8adda9f40e39312afa694ac217, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:03,417 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:15:03,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741858_1034 (size=73) 2024-11-09T09:15:03,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-09T09:15:03,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:03,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-09T09:15:03,447 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:03,826 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:03,826 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 271a1f8adda9f40e39312afa694ac217, disabling compactions & flushes 2024-11-09T09:15:03,826 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:03,826 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:03,826 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. after waiting 0 ms 2024-11-09T09:15:03,826 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:03,826 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:03,826 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 271a1f8adda9f40e39312afa694ac217: 2024-11-09T09:15:03,846 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:03,847 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731143703847"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143703847"}]},"ts":"1731143703847"} 2024-11-09T09:15:03,851 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:03,853 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:03,853 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143703853"}]},"ts":"1731143703853"} 2024-11-09T09:15:03,857 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:15:03,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=271a1f8adda9f40e39312afa694ac217, ASSIGN}] 2024-11-09T09:15:03,889 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=271a1f8adda9f40e39312afa694ac217, ASSIGN 2024-11-09T09:15:03,891 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=271a1f8adda9f40e39312afa694ac217, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-09T09:15:04,042 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=271a1f8adda9f40e39312afa694ac217, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:04,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 271a1f8adda9f40e39312afa694ac217, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:04,204 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:04,216 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:04,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 271a1f8adda9f40e39312afa694ac217, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:04,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:04,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,220 INFO [StoreOpener-271a1f8adda9f40e39312afa694ac217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,222 INFO [StoreOpener-271a1f8adda9f40e39312afa694ac217-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271a1f8adda9f40e39312afa694ac217 columnFamilyName testFamily 2024-11-09T09:15:04,222 DEBUG [StoreOpener-271a1f8adda9f40e39312afa694ac217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:04,222 INFO [StoreOpener-271a1f8adda9f40e39312afa694ac217-1 {}] regionserver.HStore(327): Store=271a1f8adda9f40e39312afa694ac217/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:04,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,226 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 271a1f8adda9f40e39312afa694ac217 2024-11-09T09:15:04,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:04,230 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 271a1f8adda9f40e39312afa694ac217; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75288109, jitterRate=0.12188024818897247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:04,231 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 271a1f8adda9f40e39312afa694ac217: 2024-11-09T09:15:04,232 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217., pid=41, masterSystemTime=1731143704204 2024-11-09T09:15:04,234 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:04,234 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:15:04,234 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=271a1f8adda9f40e39312afa694ac217, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:04,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T09:15:04,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 271a1f8adda9f40e39312afa694ac217, server=9b029db41fd9,43449,1731143603787 in 188 msec 2024-11-09T09:15:04,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-09T09:15:04,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=271a1f8adda9f40e39312afa694ac217, ASSIGN in 351 msec 2024-11-09T09:15:04,241 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:04,241 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143704241"}]},"ts":"1731143704241"} 2024-11-09T09:15:04,243 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:15:04,288 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:04,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.2940 sec 2024-11-09T09:15:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-09T09:15:06,009 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-09T09:15:06,010 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:15:06,011 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:06,020 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-09T09:15:06,020 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:06,020 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-09T09:15:06,023 DEBUG [hconnection-0x4b3ef70-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:06,024 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57920, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:06,033 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:06,036 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:15:06,039 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:15:06,039 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-09T09:15:08,091 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-09T09:15:09,241 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001772 expired clientIPAndPort=172.17.0.2:53672, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:15:09,722 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:15:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T09:15:13,056 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:15:13,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:57920 deadline: 1731143715039, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:15:13,057 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:15:13,058 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549893 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:15:13,059 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7022 2024-11-09T09:15:13,059 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5620f8e5 to 127.0.0.1:49767 2024-11-09T09:15:13,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:13,060 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:13,060 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:13,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:13,060 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20c33f6f to 127.0.0.1:49767 2024-11-09T09:15:13,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=231 (was 236), OpenFileDescriptor=450 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=31 (was 36), ProcessCount=11 (was 11), AvailableMemoryMB=3687 (was 3696) 2024-11-09T09:15:13,083 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=31, ProcessCount=11, AvailableMemoryMB=3686 2024-11-09T09:15:13,086 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34938fff to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3f0cdfa1 2024-11-09T09:15:13,122 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x34938fff {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100079c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:13,124 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ef0dd19 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a3d312a 2024-11-09T09:15:13,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f9b9e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:13,138 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:13,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T09:15:13,140 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:13,140 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:13,141 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-09T09:15:13,142 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:13,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-09T09:15:13,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741859_1035 (size=428) 2024-11-09T09:15:13,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-09T09:15:13,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:13,558 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c4fd6ba823547d504f5d98c1c026399e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:13,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741860_1036 (size=79) 2024-11-09T09:15:13,968 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:13,968 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing c4fd6ba823547d504f5d98c1c026399e, disabling compactions & flushes 2024-11-09T09:15:13,968 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:13,968 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:13,968 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. after waiting 0 ms 2024-11-09T09:15:13,968 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:13,969 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:13,969 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for c4fd6ba823547d504f5d98c1c026399e: 2024-11-09T09:15:13,970 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:13,970 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731143713970"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143713970"}]},"ts":"1731143713970"} 2024-11-09T09:15:13,973 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:13,974 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:13,974 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143713974"}]},"ts":"1731143713974"} 2024-11-09T09:15:13,977 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:15:14,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c4fd6ba823547d504f5d98c1c026399e, ASSIGN}] 2024-11-09T09:15:14,041 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c4fd6ba823547d504f5d98c1c026399e, ASSIGN 2024-11-09T09:15:14,044 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c4fd6ba823547d504f5d98c1c026399e, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-09T09:15:14,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=c4fd6ba823547d504f5d98c1c026399e, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:14,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure c4fd6ba823547d504f5d98c1c026399e, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:14,349 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:14,355 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:14,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => c4fd6ba823547d504f5d98c1c026399e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:14,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:14,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,356 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,357 INFO [StoreOpener-c4fd6ba823547d504f5d98c1c026399e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,359 INFO [StoreOpener-c4fd6ba823547d504f5d98c1c026399e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4fd6ba823547d504f5d98c1c026399e columnFamilyName testFamily 2024-11-09T09:15:14,359 DEBUG [StoreOpener-c4fd6ba823547d504f5d98c1c026399e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:14,360 INFO [StoreOpener-c4fd6ba823547d504f5d98c1c026399e-1 {}] regionserver.HStore(327): Store=c4fd6ba823547d504f5d98c1c026399e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:14,361 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,362 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:15:14,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:14,368 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened c4fd6ba823547d504f5d98c1c026399e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62945206, jitterRate=-0.062043339014053345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:14,368 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for c4fd6ba823547d504f5d98c1c026399e: 2024-11-09T09:15:14,369 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., pid=44, masterSystemTime=1731143714349 2024-11-09T09:15:14,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:14,371 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:15:14,372 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=c4fd6ba823547d504f5d98c1c026399e, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:14,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-09T09:15:14,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure c4fd6ba823547d504f5d98c1c026399e, server=9b029db41fd9,43449,1731143603787 in 177 msec 2024-11-09T09:15:14,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T09:15:14,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c4fd6ba823547d504f5d98c1c026399e, ASSIGN in 337 msec 2024-11-09T09:15:14,378 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:14,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143714378"}]},"ts":"1731143714378"} 2024-11-09T09:15:14,380 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:15:14,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:14,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.2530 sec 2024-11-09T09:15:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-09T09:15:16,152 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-09T09:15:16,152 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:15:16,153 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:16,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T09:15:16,164 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:16,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-09T09:15:16,166 DEBUG [hconnection-0x67c4ca44-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:16,167 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43114, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:16,176 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:16,178 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:15:16,179 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:15:16,180 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x34938fff {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:15:16,180 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x34938fff {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:16,181 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43116, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:15:16,183 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:15:16,251 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001770 expired clientIPAndPort=172.17.0.2:53672, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:15:19,859 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:15:20,095 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-09T09:15:22,195 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-09T09:15:22,195 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-09T09:15:22,195 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:15:22,225 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:15:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T09:15:23,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:15:23,218 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:15:23,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T09:15:23,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:24,463 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001765 expired clientIPAndPort=172.17.0.2:55788, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:15:25,182 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1965ms, rpcTimeout=1961ms 2024-11-09T09:15:25,182 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1965ms, rpcTimeout=1961ms 2024-11-09T09:15:25,183 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:15:25,183 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T09:15:22.196Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-09T09:15:25.183Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1965ms, rpcTimeout=1961ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1965ms, rpcTimeout=1961ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1965ms, rpcTimeout=1961ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T09:15:25,187 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-09T09:15:25,187 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ef0dd19 to 127.0.0.1:49767 2024-11-09T09:15:25,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:25,187 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:25,188 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:25,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:25,188 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34938fff to 127.0.0.1:49767 2024-11-09T09:15:25,201 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=226 (was 231), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=26 (was 31), ProcessCount=11 (was 11), AvailableMemoryMB=3657 (was 3686) 2024-11-09T09:15:25,209 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=26, ProcessCount=11, AvailableMemoryMB=3657 2024-11-09T09:15:25,211 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cb84ac5 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b3ad608 2024-11-09T09:15:25,272 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x6cb84ac5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5734f998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:25,276 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f4b52b2 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e64dbb3 2024-11-09T09:15:25,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a456d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:25,290 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:25,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-09T09:15:25,293 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:25,293 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:25,293 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-09T09:15:25,294 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T09:15:25,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741861_1037 (size=416) 2024-11-09T09:15:25,711 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9b80974f1b4163166250b30045e5a88b, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:25,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741862_1038 (size=67) 2024-11-09T09:15:26,124 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:26,125 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 9b80974f1b4163166250b30045e5a88b, disabling compactions & flushes 2024-11-09T09:15:26,125 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,125 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,125 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. after waiting 0 ms 2024-11-09T09:15:26,125 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,125 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,125 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9b80974f1b4163166250b30045e5a88b: 2024-11-09T09:15:26,127 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:26,127 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731143726127"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143726127"}]},"ts":"1731143726127"} 2024-11-09T09:15:26,131 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:26,132 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:26,133 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143726132"}]},"ts":"1731143726132"} 2024-11-09T09:15:26,135 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:15:26,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=9b80974f1b4163166250b30045e5a88b, ASSIGN}] 2024-11-09T09:15:26,158 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=9b80974f1b4163166250b30045e5a88b, ASSIGN 2024-11-09T09:15:26,159 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=9b80974f1b4163166250b30045e5a88b, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T09:15:26,310 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=9b80974f1b4163166250b30045e5a88b, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:26,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 9b80974f1b4163166250b30045e5a88b, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:26,470 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:26,483 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 9b80974f1b4163166250b30045e5a88b, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:26,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:26,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,487 INFO [StoreOpener-9b80974f1b4163166250b30045e5a88b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,489 INFO [StoreOpener-9b80974f1b4163166250b30045e5a88b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b80974f1b4163166250b30045e5a88b columnFamilyName testFamily 2024-11-09T09:15:26,489 DEBUG [StoreOpener-9b80974f1b4163166250b30045e5a88b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:26,490 INFO [StoreOpener-9b80974f1b4163166250b30045e5a88b-1 {}] regionserver.HStore(327): Store=9b80974f1b4163166250b30045e5a88b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:26,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,494 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 9b80974f1b4163166250b30045e5a88b 2024-11-09T09:15:26,498 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:26,498 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 9b80974f1b4163166250b30045e5a88b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67446361, jitterRate=0.0050290971994400024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:26,499 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 9b80974f1b4163166250b30045e5a88b: 2024-11-09T09:15:26,500 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b., pid=47, masterSystemTime=1731143726469 2024-11-09T09:15:26,502 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,502 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:15:26,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=9b80974f1b4163166250b30045e5a88b, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:26,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T09:15:26,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 9b80974f1b4163166250b30045e5a88b, server=9b029db41fd9,43449,1731143603787 in 190 msec 2024-11-09T09:15:26,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-09T09:15:26,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=9b80974f1b4163166250b30045e5a88b, ASSIGN in 349 msec 2024-11-09T09:15:26,509 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:26,509 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143726509"}]},"ts":"1731143726509"} 2024-11-09T09:15:26,511 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:15:26,554 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:26,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.2640 sec 2024-11-09T09:15:27,067 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001759 expired clientIPAndPort=172.17.0.2:44398, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:15:27,067 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001758 expired clientIPAndPort=172.17.0.2:44398, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:15:27,067 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001761 expired clientIPAndPort=172.17.0.2:44398, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:15:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T09:15:28,303 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-09T09:15:28,303 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:15:28,304 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:28,314 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-09T09:15:28,315 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:28,315 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-09T09:15:28,317 DEBUG [hconnection-0x2f93630-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:28,318 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38854, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:28,326 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:28,331 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16981ff6 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60529ccc 2024-11-09T09:15:28,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d43fae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:28,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:15:28,374 DEBUG [hconnection-0x2903b97e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:28,375 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38864, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:28,380 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:15:28,381 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549906 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:28,382 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:15:28,383 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549906 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:28,383 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-09T09:15:28,835 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:15:29,720 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:15:30,059 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:15:32,024 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:15:32,098 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-09T09:15:32,275 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001752 expired clientIPAndPort=172.17.0.2:44088, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:15:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-09T09:15:33,447 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:34,393 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T09:15:34.393Z, RpcRetryingCaller{globalStartTime=2024-11-09T09:15:28.383Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T09:15:34.393Z, RpcRetryingCaller{globalStartTime=2024-11-09T09:15:28.383Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:15:34,394 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6011 2024-11-09T09:15:34,395 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x16981ff6 to 127.0.0.1:49767 2024-11-09T09:15:34,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:34,395 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f4b52b2 to 127.0.0.1:49767 2024-11-09T09:15:34,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:34,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:34,396 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:34,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:34,397 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6cb84ac5 to 127.0.0.1:49767 2024-11-09T09:15:34,408 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=232 (was 226) - Thread LEAK? -, OpenFileDescriptor=452 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=22 (was 26), ProcessCount=11 (was 11), AvailableMemoryMB=3614 (was 3657) 2024-11-09T09:15:34,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=232, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=22, ProcessCount=11, AvailableMemoryMB=3614 2024-11-09T09:15:34,419 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13006856 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2664ae92 2024-11-09T09:15:34,446 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x13006856 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc94600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:34,447 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27e9399e to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27e8e565 2024-11-09T09:15:34,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ba7b16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:34,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T09:15:34,462 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:34,462 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:34,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-09T09:15:34,463 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-09T09:15:34,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741863_1039 (size=432) 2024-11-09T09:15:34,877 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 36ec248a87a5f6a35a3be224ec1de4c5, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:34,884 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:15:34,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741864_1040 (size=83) 2024-11-09T09:15:35,296 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:35,296 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 36ec248a87a5f6a35a3be224ec1de4c5, disabling compactions & flushes 2024-11-09T09:15:35,296 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,296 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. after waiting 0 ms 2024-11-09T09:15:35,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,297 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 36ec248a87a5f6a35a3be224ec1de4c5: 2024-11-09T09:15:35,300 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:35,301 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731143735300"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143735300"}]},"ts":"1731143735300"} 2024-11-09T09:15:35,303 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:35,304 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:35,304 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143735304"}]},"ts":"1731143735304"} 2024-11-09T09:15:35,306 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T09:15:35,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=36ec248a87a5f6a35a3be224ec1de4c5, ASSIGN}] 2024-11-09T09:15:35,355 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=36ec248a87a5f6a35a3be224ec1de4c5, ASSIGN 2024-11-09T09:15:35,356 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=36ec248a87a5f6a35a3be224ec1de4c5, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-09T09:15:35,507 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=36ec248a87a5f6a35a3be224ec1de4c5, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:35,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 36ec248a87a5f6a35a3be224ec1de4c5, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:35,665 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:35,669 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,669 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => 36ec248a87a5f6a35a3be224ec1de4c5, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:35,670 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,670 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:35,670 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for 36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,670 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for 36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,672 INFO [StoreOpener-36ec248a87a5f6a35a3be224ec1de4c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,674 INFO [StoreOpener-36ec248a87a5f6a35a3be224ec1de4c5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36ec248a87a5f6a35a3be224ec1de4c5 columnFamilyName testFamily 2024-11-09T09:15:35,675 DEBUG [StoreOpener-36ec248a87a5f6a35a3be224ec1de4c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:35,675 INFO [StoreOpener-36ec248a87a5f6a35a3be224ec1de4c5-1 {}] regionserver.HStore(327): Store=36ec248a87a5f6a35a3be224ec1de4c5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:35,677 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,677 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,680 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for 36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:15:35,683 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:35,684 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened 36ec248a87a5f6a35a3be224ec1de4c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73406684, jitterRate=0.09384483098983765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:35,684 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for 36ec248a87a5f6a35a3be224ec1de4c5: 2024-11-09T09:15:35,685 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., pid=50, masterSystemTime=1731143735665 2024-11-09T09:15:35,687 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,687 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:15:35,688 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=36ec248a87a5f6a35a3be224ec1de4c5, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:35,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-09T09:15:35,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 36ec248a87a5f6a35a3be224ec1de4c5, server=9b029db41fd9,43449,1731143603787 in 180 msec 2024-11-09T09:15:35,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T09:15:35,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=36ec248a87a5f6a35a3be224ec1de4c5, ASSIGN in 339 msec 2024-11-09T09:15:35,694 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:35,694 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143735694"}]},"ts":"1731143735694"} 2024-11-09T09:15:35,696 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T09:15:35,737 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:35,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.2770 sec 2024-11-09T09:15:36,882 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001747 expired clientIPAndPort=172.17.0.2:44098, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:15:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-09T09:15:37,470 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-09T09:15:37,470 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T09:15:37,471 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:37,483 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T09:15:37,483 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:37,483 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-09T09:15:37,485 DEBUG [hconnection-0x5ec702fe-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:37,486 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38878, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:37,496 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:37,499 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:15:37,499 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:15:37,501 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x13006856 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:15:37,501 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x13006856 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:37,503 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38892, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:15:37,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:15:38,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-09T09:15:38,513 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-09T09:15:38,513 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-09T09:15:38,513 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:15:38,885 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001745 expired clientIPAndPort=172.17.0.2:44098, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:15:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T09:15:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:15:39,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:15:40,502 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=976ms 2024-11-09T09:15:40,502 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=976ms 2024-11-09T09:15:40,502 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:15:40,502 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T09:15:38.514Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-09T09:15:40.502Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=976ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T09:15:40,503 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2024-11-09T09:15:40,504 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x27e9399e to 127.0.0.1:49767 2024-11-09T09:15:40,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:40,504 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:40,504 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:40,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:40,504 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13006856 to 127.0.0.1:49767 2024-11-09T09:15:40,511 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 232), OpenFileDescriptor=448 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=20 (was 22), ProcessCount=11 (was 11), AvailableMemoryMB=3603 (was 3614) 2024-11-09T09:15:40,520 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=20, ProcessCount=11, AvailableMemoryMB=3603 2024-11-09T09:15:40,521 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e10c787 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2409815 2024-11-09T09:15:40,588 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7e10c787 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e96f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:40,591 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66f2fa9f to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68a205a6 2024-11-09T09:15:40,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1076cbca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:40,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-09T09:15:40,614 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:40,614 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:40,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-09T09:15:40,616 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T09:15:40,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741865_1041 (size=436) 2024-11-09T09:15:41,024 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:15:41,031 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ef34766cd3a29a746f1b7db5199d51e4, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:41,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741866_1042 (size=87) 2024-11-09T09:15:41,042 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:41,042 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing ef34766cd3a29a746f1b7db5199d51e4, disabling compactions & flushes 2024-11-09T09:15:41,043 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,043 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,043 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. after waiting 0 ms 2024-11-09T09:15:41,043 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,043 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,043 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for ef34766cd3a29a746f1b7db5199d51e4: 2024-11-09T09:15:41,044 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:41,045 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731143741044"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143741044"}]},"ts":"1731143741044"} 2024-11-09T09:15:41,047 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:41,048 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:41,048 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143741048"}]},"ts":"1731143741048"} 2024-11-09T09:15:41,049 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:15:41,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=ef34766cd3a29a746f1b7db5199d51e4, ASSIGN}] 2024-11-09T09:15:41,122 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=ef34766cd3a29a746f1b7db5199d51e4, ASSIGN 2024-11-09T09:15:41,124 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=ef34766cd3a29a746f1b7db5199d51e4, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:41,172 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:15:41,275 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=ef34766cd3a29a746f1b7db5199d51e4, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:41,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure ef34766cd3a29a746f1b7db5199d51e4, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:41,429 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:41,435 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,435 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => ef34766cd3a29a746f1b7db5199d51e4, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:41,435 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,435 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:41,436 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,436 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,438 INFO [StoreOpener-ef34766cd3a29a746f1b7db5199d51e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,441 INFO [StoreOpener-ef34766cd3a29a746f1b7db5199d51e4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef34766cd3a29a746f1b7db5199d51e4 columnFamilyName testFamily 2024-11-09T09:15:41,441 DEBUG [StoreOpener-ef34766cd3a29a746f1b7db5199d51e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:41,442 INFO [StoreOpener-ef34766cd3a29a746f1b7db5199d51e4-1 {}] regionserver.HStore(327): Store=ef34766cd3a29a746f1b7db5199d51e4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:41,444 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,444 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,448 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:15:41,451 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:41,452 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened ef34766cd3a29a746f1b7db5199d51e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73336260, jitterRate=0.09279543161392212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:41,453 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for ef34766cd3a29a746f1b7db5199d51e4: 2024-11-09T09:15:41,454 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4., pid=53, masterSystemTime=1731143741429 2024-11-09T09:15:41,456 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,456 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:15:41,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=ef34766cd3a29a746f1b7db5199d51e4, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:41,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T09:15:41,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure ef34766cd3a29a746f1b7db5199d51e4, server=9b029db41fd9,43449,1731143603787 in 181 msec 2024-11-09T09:15:41,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-09T09:15:41,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=ef34766cd3a29a746f1b7db5199d51e4, ASSIGN in 340 msec 2024-11-09T09:15:41,462 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:41,463 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143741462"}]},"ts":"1731143741462"} 2024-11-09T09:15:41,464 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:15:41,483 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:41,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 872 msec 2024-11-09T09:15:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T09:15:41,621 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-09T09:15:41,621 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:15:41,622 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:41,631 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T09:15:41,631 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:41,631 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-09T09:15:41,634 DEBUG [hconnection-0x395e496e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:41,636 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57280, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:41,647 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:41,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T09:15:41,650 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:15:41,652 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7e10c787 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:15:41,652 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7e10c787 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:41,654 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57284, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:15:41,657 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T09:15:41,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549918 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:41,658 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T09:15:41,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549918 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:41,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001698 2024-11-09T09:15:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:57284 deadline: 1731143744657, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:15:41,659 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549918 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:15:41,660 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T09:15:41,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549919 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:41,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T09:15:41,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:15:41,662 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:15:41,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549920 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:41,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001696 2024-11-09T09:15:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:57284 deadline: 1731143744661, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:15:41,663 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549920 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:15:41,664 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:15:41,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549921 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:41,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5312316530116001695 2024-11-09T09:15:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:57284 deadline: 1731143744663, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:15:41,665 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549921 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:15:41,666 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:15:41,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549922 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:41,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5312316530116001694 2024-11-09T09:15:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:57284 deadline: 1731143744665, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:15:41,667 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549922 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:15:41,668 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:15:41,668 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66f2fa9f to 127.0.0.1:49767 2024-11-09T09:15:41,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:41,669 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:41,669 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:41,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:41,669 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e10c787 to 127.0.0.1:49767 2024-11-09T09:15:41,676 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=20 (was 20), ProcessCount=11 (was 11), AvailableMemoryMB=3580 (was 3603) 2024-11-09T09:15:41,684 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=20, ProcessCount=11, AvailableMemoryMB=3580 2024-11-09T09:15:41,686 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ffd1a65 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9dd740b 2024-11-09T09:15:41,695 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x5ffd1a65 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33203625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:41,696 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e798a61 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ab094b4 2024-11-09T09:15:41,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da99db5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:41,710 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:41,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-09T09:15:41,712 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:41,712 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:41,713 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:41,713 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-09T09:15:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-09T09:15:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741867_1043 (size=422) 2024-11-09T09:15:42,128 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d29da8fc7e2c0292000b4931ecdf1ed1, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741868_1044 (size=73) 2024-11-09T09:15:42,543 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:42,543 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing d29da8fc7e2c0292000b4931ecdf1ed1, disabling compactions & flushes 2024-11-09T09:15:42,543 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,544 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,544 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. after waiting 0 ms 2024-11-09T09:15:42,544 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,544 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,544 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for d29da8fc7e2c0292000b4931ecdf1ed1: 2024-11-09T09:15:42,546 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:42,547 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731143742546"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143742546"}]},"ts":"1731143742546"} 2024-11-09T09:15:42,550 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:42,552 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:42,552 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143742552"}]},"ts":"1731143742552"} 2024-11-09T09:15:42,555 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:15:42,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=d29da8fc7e2c0292000b4931ecdf1ed1, ASSIGN}] 2024-11-09T09:15:42,606 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=d29da8fc7e2c0292000b4931ecdf1ed1, ASSIGN 2024-11-09T09:15:42,607 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=d29da8fc7e2c0292000b4931ecdf1ed1, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-09T09:15:42,758 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=d29da8fc7e2c0292000b4931ecdf1ed1, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:42,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure d29da8fc7e2c0292000b4931ecdf1ed1, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:42,918 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:42,926 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,926 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => d29da8fc7e2c0292000b4931ecdf1ed1, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:42,927 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,927 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:42,927 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,927 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,930 INFO [StoreOpener-d29da8fc7e2c0292000b4931ecdf1ed1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,933 INFO [StoreOpener-d29da8fc7e2c0292000b4931ecdf1ed1-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d29da8fc7e2c0292000b4931ecdf1ed1 columnFamilyName testFamily 2024-11-09T09:15:42,933 DEBUG [StoreOpener-d29da8fc7e2c0292000b4931ecdf1ed1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:42,934 INFO [StoreOpener-d29da8fc7e2c0292000b4931ecdf1ed1-1 {}] regionserver.HStore(327): Store=d29da8fc7e2c0292000b4931ecdf1ed1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:42,935 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,935 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,938 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:15:42,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:42,941 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened d29da8fc7e2c0292000b4931ecdf1ed1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59940301, jitterRate=-0.10681991279125214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:42,942 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for d29da8fc7e2c0292000b4931ecdf1ed1: 2024-11-09T09:15:42,943 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1., pid=56, masterSystemTime=1731143742918 2024-11-09T09:15:42,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,944 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:15:42,945 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=d29da8fc7e2c0292000b4931ecdf1ed1, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:42,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-09T09:15:42,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure d29da8fc7e2c0292000b4931ecdf1ed1, server=9b029db41fd9,43449,1731143603787 in 183 msec 2024-11-09T09:15:42,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T09:15:42,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=d29da8fc7e2c0292000b4931ecdf1ed1, ASSIGN in 345 msec 2024-11-09T09:15:42,950 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:42,950 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143742950"}]},"ts":"1731143742950"} 2024-11-09T09:15:42,951 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:15:42,962 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:42,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.2520 sec 2024-11-09T09:15:43,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-09T09:15:43,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:43,449 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-09T09:15:43,449 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:43,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T09:15:43,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T09:15:44,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-09T09:15:44,107 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-09T09:15:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-09T09:15:44,723 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-09T09:15:44,723 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:15:44,724 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:44,734 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-09T09:15:44,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:44,735 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-09T09:15:44,737 DEBUG [hconnection-0x42f03df5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:44,738 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57300, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:44,746 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:44,748 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:15:44,749 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:15:44,993 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001740 expired clientIPAndPort=172.17.0.2:48090, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:15:46,940 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:15:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T09:15:51,764 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:15:52,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:15:53,754 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:15:53,755 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T09:15:53.755Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1990ms, rpcTimeout=1986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:15:53,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-09T09:15:53,759 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0e798a61 to 127.0.0.1:49767 2024-11-09T09:15:53,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:53,759 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:15:53,759 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:15:53,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:15:53,759 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ffd1a65 to 127.0.0.1:49767 2024-11-09T09:15:53,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=228 (was 233), OpenFileDescriptor=446 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=29 (was 20) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3546 (was 3580) 2024-11-09T09:15:53,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=228, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=29, ProcessCount=11, AvailableMemoryMB=3545 2024-11-09T09:15:53,785 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e4aaa7e to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c486ff2 2024-11-09T09:15:53,804 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x0e4aaa7e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c68d666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:53,806 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ba65f32 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d19ed7b 2024-11-09T09:15:53,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dba5fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:15:53,822 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:15:53,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-09T09:15:53,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:15:53,823 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:53,824 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-09T09:15:53,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:15:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T09:15:53,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741869_1045 (size=420) 2024-11-09T09:15:54,241 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2fb42412d02f8c07a7a7398fa2e5222e, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:15:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741870_1046 (size=71) 2024-11-09T09:15:54,655 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:54,656 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 2fb42412d02f8c07a7a7398fa2e5222e, disabling compactions & flushes 2024-11-09T09:15:54,656 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:54,656 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:54,656 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. after waiting 0 ms 2024-11-09T09:15:54,657 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:54,657 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:54,657 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2fb42412d02f8c07a7a7398fa2e5222e: 2024-11-09T09:15:54,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:15:54,660 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731143754660"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143754660"}]},"ts":"1731143754660"} 2024-11-09T09:15:54,664 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:15:54,664 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:15:54,665 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143754665"}]},"ts":"1731143754665"} 2024-11-09T09:15:54,666 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:15:54,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=2fb42412d02f8c07a7a7398fa2e5222e, ASSIGN}] 2024-11-09T09:15:54,712 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=2fb42412d02f8c07a7a7398fa2e5222e, ASSIGN 2024-11-09T09:15:54,712 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=2fb42412d02f8c07a7a7398fa2e5222e, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:15:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T09:15:54,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=2fb42412d02f8c07a7a7398fa2e5222e, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:54,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 2fb42412d02f8c07a7a7398fa2e5222e, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:15:55,017 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:15:55,022 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:55,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => 2fb42412d02f8c07a7a7398fa2e5222e, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:15:55,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:15:55,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,025 INFO [StoreOpener-2fb42412d02f8c07a7a7398fa2e5222e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,027 INFO [StoreOpener-2fb42412d02f8c07a7a7398fa2e5222e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fb42412d02f8c07a7a7398fa2e5222e columnFamilyName testFamily 2024-11-09T09:15:55,027 DEBUG [StoreOpener-2fb42412d02f8c07a7a7398fa2e5222e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:15:55,028 INFO [StoreOpener-2fb42412d02f8c07a7a7398fa2e5222e-1 {}] regionserver.HStore(327): Store=2fb42412d02f8c07a7a7398fa2e5222e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:15:55,029 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,029 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:15:55,035 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:15:55,035 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened 2fb42412d02f8c07a7a7398fa2e5222e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72695971, jitterRate=0.0832543820142746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:15:55,036 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for 2fb42412d02f8c07a7a7398fa2e5222e: 2024-11-09T09:15:55,037 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e., pid=59, masterSystemTime=1731143755017 2024-11-09T09:15:55,038 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:55,039 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:15:55,039 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=2fb42412d02f8c07a7a7398fa2e5222e, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:15:55,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T09:15:55,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 2fb42412d02f8c07a7a7398fa2e5222e, server=9b029db41fd9,43449,1731143603787 in 176 msec 2024-11-09T09:15:55,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T09:15:55,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=2fb42412d02f8c07a7a7398fa2e5222e, ASSIGN in 332 msec 2024-11-09T09:15:55,045 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:15:55,045 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143755045"}]},"ts":"1731143755045"} 2024-11-09T09:15:55,046 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:15:55,087 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:15:55,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 1.2650 sec 2024-11-09T09:15:56,114 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-09T09:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T09:15:56,841 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-09T09:15:56,841 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:15:56,842 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:56,846 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T09:15:56,846 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:15:56,846 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-09T09:15:56,848 DEBUG [hconnection-0x2b094a55-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:56,850 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49746, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:56,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:15:56,881 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:15:56,881 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:15:56,883 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x0e4aaa7e {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:15:56,883 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x0e4aaa7e {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:15:56,885 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49752, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:15:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:15:56,889 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:15:56,889 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549936 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:56,890 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:15:56,891 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549936 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:15:56,891 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-09T09:15:58,265 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:00,567 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:03,419 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001727 expired clientIPAndPort=172.17.0.2:57910, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:16:03,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-09T09:16:03,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:04,471 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:16:05,687 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:16:05,892 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9001 2024-11-09T09:16:05,893 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ba65f32 to 127.0.0.1:49767 2024-11-09T09:16:05,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:05,893 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549936 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:16:05,893 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:05,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001680' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T09:16:05,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:05,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:05,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:49752 deadline: 1731143771892, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001680' does not exist 2024-11-09T09:16:05,894 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0e4aaa7e to 127.0.0.1:49767 2024-11-09T09:16:05,894 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:05,894 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9b029db41fd9,43449,1731143603787 for closing scanner id = -5312316530116001680 for 2fb42412d02f8c07a7a7398fa2e5222e of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T09:16:05,907 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=226 (was 228), OpenFileDescriptor=444 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=24 (was 29), ProcessCount=11 (was 11), AvailableMemoryMB=3484 (was 3545) 2024-11-09T09:16:05,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=24, ProcessCount=11, AvailableMemoryMB=3484 2024-11-09T09:16:05,919 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x787cf01c to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11bc10e8 2024-11-09T09:16:05,954 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x787cf01c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f0826a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:05,957 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d02585e to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15c28b53 2024-11-09T09:16:05,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a2165e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:05,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-09T09:16:05,975 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:05,975 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:05,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-09T09:16:05,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-09T09:16:05,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741871_1047 (size=431) 2024-11-09T09:16:06,393 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:06,395 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3a971eb4fc571ad608de182c691878da, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:06,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741872_1048 (size=82) 2024-11-09T09:16:06,807 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:06,808 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 3a971eb4fc571ad608de182c691878da, disabling compactions & flushes 2024-11-09T09:16:06,808 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:06,808 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:06,808 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. after waiting 0 ms 2024-11-09T09:16:06,808 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:06,808 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:06,808 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3a971eb4fc571ad608de182c691878da: 2024-11-09T09:16:06,809 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:06,809 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731143766809"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143766809"}]},"ts":"1731143766809"} 2024-11-09T09:16:06,811 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:06,812 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:06,812 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143766812"}]},"ts":"1731143766812"} 2024-11-09T09:16:06,814 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:06,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=3a971eb4fc571ad608de182c691878da, ASSIGN}] 2024-11-09T09:16:06,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=3a971eb4fc571ad608de182c691878da, ASSIGN 2024-11-09T09:16:06,862 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=3a971eb4fc571ad608de182c691878da, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-09T09:16:07,012 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=3a971eb4fc571ad608de182c691878da, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:07,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 3a971eb4fc571ad608de182c691878da, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:07,170 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:07,182 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:07,183 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 3a971eb4fc571ad608de182c691878da, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:07,183 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,183 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:07,184 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,184 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,186 INFO [StoreOpener-3a971eb4fc571ad608de182c691878da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,188 INFO [StoreOpener-3a971eb4fc571ad608de182c691878da-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a971eb4fc571ad608de182c691878da columnFamilyName testFamily 2024-11-09T09:16:07,188 DEBUG [StoreOpener-3a971eb4fc571ad608de182c691878da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:07,189 INFO [StoreOpener-3a971eb4fc571ad608de182c691878da-1 {}] regionserver.HStore(327): Store=3a971eb4fc571ad608de182c691878da/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:07,190 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,190 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,192 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 3a971eb4fc571ad608de182c691878da 2024-11-09T09:16:07,195 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:07,195 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 3a971eb4fc571ad608de182c691878da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66176557, jitterRate=-0.013892456889152527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:07,196 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 3a971eb4fc571ad608de182c691878da: 2024-11-09T09:16:07,197 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da., pid=62, masterSystemTime=1731143767170 2024-11-09T09:16:07,199 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:07,199 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:16:07,199 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=3a971eb4fc571ad608de182c691878da, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:07,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-09T09:16:07,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 3a971eb4fc571ad608de182c691878da, server=9b029db41fd9,43449,1731143603787 in 184 msec 2024-11-09T09:16:07,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T09:16:07,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=3a971eb4fc571ad608de182c691878da, ASSIGN in 342 msec 2024-11-09T09:16:07,204 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:07,204 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143767204"}]},"ts":"1731143767204"} 2024-11-09T09:16:07,206 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:07,219 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:07,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.2460 sec 2024-11-09T09:16:08,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-09T09:16:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-09T09:16:08,984 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-09T09:16:08,984 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:08,984 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:08,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:08,995 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:08,995 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-09T09:16:08,998 DEBUG [hconnection-0x4482a059-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:08,999 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59194, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:09,007 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:09,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T09:16:09,011 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T09:16:09,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549940 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:09,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T09:16:09,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549940 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:09,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001676 2024-11-09T09:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59194 deadline: 1731143772013, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:16:09,014 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:16:09,017 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T09:16:09,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549941 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:09,018 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T09:16:09,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:16:09,020 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:16:09,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549942 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:09,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5312316530116001674 2024-11-09T09:16:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59194 deadline: 1731143772020, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:16:09,022 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:16:09,023 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:16:09,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549943 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:09,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5312316530116001673 2024-11-09T09:16:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59194 deadline: 1731143772023, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:16:09,025 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:16:09,026 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:16:09,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549944 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:09,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5312316530116001672 2024-11-09T09:16:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59194 deadline: 1731143772027, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T09:16:09,028 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:16:09,031 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T09:16:09,032 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d02585e to 127.0.0.1:49767 2024-11-09T09:16:09,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:09,032 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:09,032 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:09,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:09,033 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x787cf01c to 127.0.0.1:49767 2024-11-09T09:16:09,043 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=234 (was 226) - Thread LEAK? -, OpenFileDescriptor=450 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=22 (was 24), ProcessCount=11 (was 11), AvailableMemoryMB=3468 (was 3484) 2024-11-09T09:16:09,050 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=22, ProcessCount=11, AvailableMemoryMB=3468 2024-11-09T09:16:09,052 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5805bb02 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f114fbc 2024-11-09T09:16:09,061 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x5805bb02 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d0fd74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:09,062 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b160b0b to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ad1c166 2024-11-09T09:16:09,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f79b895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:09,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-09T09:16:09,074 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:09,074 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:09,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-09T09:16:09,075 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T09:16:09,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741873_1049 (size=425) 2024-11-09T09:16:09,487 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 316c4850c2b79d7cbe87367fe6cd94b3, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:09,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741874_1050 (size=76) 2024-11-09T09:16:09,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:09,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 316c4850c2b79d7cbe87367fe6cd94b3, disabling compactions & flushes 2024-11-09T09:16:09,905 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:09,905 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:09,905 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. after waiting 0 ms 2024-11-09T09:16:09,905 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:09,905 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:09,905 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 316c4850c2b79d7cbe87367fe6cd94b3: 2024-11-09T09:16:09,907 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:09,908 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731143769907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143769907"}]},"ts":"1731143769907"} 2024-11-09T09:16:09,911 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:09,913 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:09,913 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143769913"}]},"ts":"1731143769913"} 2024-11-09T09:16:09,916 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:09,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=316c4850c2b79d7cbe87367fe6cd94b3, ASSIGN}] 2024-11-09T09:16:09,961 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=316c4850c2b79d7cbe87367fe6cd94b3, ASSIGN 2024-11-09T09:16:09,962 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=316c4850c2b79d7cbe87367fe6cd94b3, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T09:16:10,112 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=316c4850c2b79d7cbe87367fe6cd94b3, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:10,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure 316c4850c2b79d7cbe87367fe6cd94b3, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:10,266 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:10,271 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:10,271 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => 316c4850c2b79d7cbe87367fe6cd94b3, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:10,271 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true 316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,272 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:10,272 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for 316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,272 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for 316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,274 INFO [StoreOpener-316c4850c2b79d7cbe87367fe6cd94b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,276 INFO [StoreOpener-316c4850c2b79d7cbe87367fe6cd94b3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 316c4850c2b79d7cbe87367fe6cd94b3 columnFamilyName testFamily 2024-11-09T09:16:10,276 DEBUG [StoreOpener-316c4850c2b79d7cbe87367fe6cd94b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:10,277 INFO [StoreOpener-316c4850c2b79d7cbe87367fe6cd94b3-1 {}] regionserver.HStore(327): Store=316c4850c2b79d7cbe87367fe6cd94b3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:10,278 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,278 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,282 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for 316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:16:10,285 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:10,285 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened 316c4850c2b79d7cbe87367fe6cd94b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74933289, jitterRate=0.11659301817417145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:10,286 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for 316c4850c2b79d7cbe87367fe6cd94b3: 2024-11-09T09:16:10,287 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3., pid=65, masterSystemTime=1731143770266 2024-11-09T09:16:10,289 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:10,289 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:16:10,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=316c4850c2b79d7cbe87367fe6cd94b3, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:10,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T09:16:10,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure 316c4850c2b79d7cbe87367fe6cd94b3, server=9b029db41fd9,43449,1731143603787 in 177 msec 2024-11-09T09:16:10,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-09T09:16:10,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=316c4850c2b79d7cbe87367fe6cd94b3, ASSIGN in 333 msec 2024-11-09T09:16:10,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:10,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143770295"}]},"ts":"1731143770295"} 2024-11-09T09:16:10,297 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:10,308 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:10,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.2360 sec 2024-11-09T09:16:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T09:16:12,083 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-09T09:16:12,083 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:12,083 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:12,092 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:12,092 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:12,092 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-09T09:16:12,095 DEBUG [hconnection-0x1724f6ec-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:12,096 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59210, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:12,106 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:12,109 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:16:12,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:16:12,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-09T09:16:12,723 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:13,132 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001723 expired clientIPAndPort=172.17.0.2:57920, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:16:13,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-09T09:16:13,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:13,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-09T09:16:13,447 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T09:16:14,123 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-09T09:16:14,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:16:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59210 deadline: 1731143775111, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:16:14,131 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:16:14,131 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549949 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:16:14,133 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2023 2024-11-09T09:16:14,133 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b160b0b to 127.0.0.1:49767 2024-11-09T09:16:14,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:14,133 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:14,134 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:14,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:14,134 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5805bb02 to 127.0.0.1:49767 2024-11-09T09:16:14,145 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234 (was 234), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=20 (was 22), ProcessCount=11 (was 11), AvailableMemoryMB=3429 (was 3468) 2024-11-09T09:16:14,154 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=234, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=20, ProcessCount=11, AvailableMemoryMB=3428 2024-11-09T09:16:14,156 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e1c0b92 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5637474b 2024-11-09T09:16:14,203 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x3e1c0b92 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a158c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:14,206 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a5f9574 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b78a4d1 2024-11-09T09:16:14,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45667cb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:14,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-09T09:16:14,228 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:14,228 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:14,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-09T09:16:14,229 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-09T09:16:14,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741875_1051 (size=426) 2024-11-09T09:16:14,642 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4622f677b70a31e1a9ad71877e78cdbc, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:14,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741876_1052 (size=77) 2024-11-09T09:16:15,054 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:15,054 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 4622f677b70a31e1a9ad71877e78cdbc, disabling compactions & flushes 2024-11-09T09:16:15,054 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,054 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,054 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. after waiting 0 ms 2024-11-09T09:16:15,054 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,054 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,054 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4622f677b70a31e1a9ad71877e78cdbc: 2024-11-09T09:16:15,055 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:15,055 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731143775055"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143775055"}]},"ts":"1731143775055"} 2024-11-09T09:16:15,057 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:15,058 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:15,058 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143775058"}]},"ts":"1731143775058"} 2024-11-09T09:16:15,059 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:15,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=4622f677b70a31e1a9ad71877e78cdbc, ASSIGN}] 2024-11-09T09:16:15,102 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=4622f677b70a31e1a9ad71877e78cdbc, ASSIGN 2024-11-09T09:16:15,103 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=4622f677b70a31e1a9ad71877e78cdbc, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-09T09:16:15,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=4622f677b70a31e1a9ad71877e78cdbc, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:15,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 4622f677b70a31e1a9ad71877e78cdbc, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:15,412 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:15,423 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,423 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 4622f677b70a31e1a9ad71877e78cdbc, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:15,423 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,423 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:15,424 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,424 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,427 INFO [StoreOpener-4622f677b70a31e1a9ad71877e78cdbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,429 INFO [StoreOpener-4622f677b70a31e1a9ad71877e78cdbc-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4622f677b70a31e1a9ad71877e78cdbc columnFamilyName testFamily 2024-11-09T09:16:15,429 DEBUG [StoreOpener-4622f677b70a31e1a9ad71877e78cdbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:15,430 INFO [StoreOpener-4622f677b70a31e1a9ad71877e78cdbc-1 {}] regionserver.HStore(327): Store=4622f677b70a31e1a9ad71877e78cdbc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:15,431 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,432 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,434 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:16:15,437 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:15,437 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 4622f677b70a31e1a9ad71877e78cdbc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71957287, jitterRate=0.07224713265895844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:15,438 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 4622f677b70a31e1a9ad71877e78cdbc: 2024-11-09T09:16:15,439 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc., pid=68, masterSystemTime=1731143775411 2024-11-09T09:16:15,440 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,440 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:16:15,441 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=4622f677b70a31e1a9ad71877e78cdbc, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:15,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-09T09:16:15,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 4622f677b70a31e1a9ad71877e78cdbc, server=9b029db41fd9,43449,1731143603787 in 185 msec 2024-11-09T09:16:15,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T09:16:15,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=4622f677b70a31e1a9ad71877e78cdbc, ASSIGN in 342 msec 2024-11-09T09:16:15,445 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:15,445 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143775445"}]},"ts":"1731143775445"} 2024-11-09T09:16:15,446 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:15,485 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:15,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.2600 sec 2024-11-09T09:16:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-09T09:16:17,236 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-09T09:16:17,237 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:17,237 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:17,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:17,245 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:17,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-09T09:16:17,247 DEBUG [hconnection-0x17109c60-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:17,249 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59212, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:17,257 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:17,259 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:16:17,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:16:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T09:16:19,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:16:20,127 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-09T09:16:20,263 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:16:20,264 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T09:16:20.263Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=984ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:16:20,266 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3006 2024-11-09T09:16:20,266 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a5f9574 to 127.0.0.1:49767 2024-11-09T09:16:20,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:20,267 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:20,267 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:20,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:20,267 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3e1c0b92 to 127.0.0.1:49767 2024-11-09T09:16:20,280 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=231 (was 234), OpenFileDescriptor=451 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=19 (was 20), ProcessCount=11 (was 11), AvailableMemoryMB=3413 (was 3428) 2024-11-09T09:16:20,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=19, ProcessCount=11, AvailableMemoryMB=3413 2024-11-09T09:16:20,293 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09212546 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13e362e4 2024-11-09T09:16:20,362 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x09212546 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3a1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:20,365 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4752c8a8 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f026193 2024-11-09T09:16:20,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f096020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:20,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-09T09:16:20,382 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:20,382 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:20,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-09T09:16:20,383 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-09T09:16:20,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741877_1053 (size=424) 2024-11-09T09:16:20,780 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:20,803 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d8faed9fa602af320f8ab06bb017e46d, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:20,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741878_1054 (size=75) 2024-11-09T09:16:20,926 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:21,217 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:21,217 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing d8faed9fa602af320f8ab06bb017e46d, disabling compactions & flushes 2024-11-09T09:16:21,217 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,218 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,218 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. after waiting 0 ms 2024-11-09T09:16:21,218 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,218 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,218 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for d8faed9fa602af320f8ab06bb017e46d: 2024-11-09T09:16:21,221 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:21,221 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731143781221"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143781221"}]},"ts":"1731143781221"} 2024-11-09T09:16:21,226 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:21,228 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:21,229 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143781228"}]},"ts":"1731143781228"} 2024-11-09T09:16:21,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:21,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=d8faed9fa602af320f8ab06bb017e46d, ASSIGN}] 2024-11-09T09:16:21,295 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=d8faed9fa602af320f8ab06bb017e46d, ASSIGN 2024-11-09T09:16:21,296 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=d8faed9fa602af320f8ab06bb017e46d, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-09T09:16:21,446 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=d8faed9fa602af320f8ab06bb017e46d, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:21,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure d8faed9fa602af320f8ab06bb017e46d, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:21,600 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:21,608 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,609 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => d8faed9fa602af320f8ab06bb017e46d, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:21,610 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,610 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:21,610 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,610 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,614 INFO [StoreOpener-d8faed9fa602af320f8ab06bb017e46d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,617 INFO [StoreOpener-d8faed9fa602af320f8ab06bb017e46d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8faed9fa602af320f8ab06bb017e46d columnFamilyName testFamily 2024-11-09T09:16:21,617 DEBUG [StoreOpener-d8faed9fa602af320f8ab06bb017e46d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:21,618 INFO [StoreOpener-d8faed9fa602af320f8ab06bb017e46d-1 {}] regionserver.HStore(327): Store=d8faed9fa602af320f8ab06bb017e46d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:21,619 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,620 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,623 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:16:21,626 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:21,627 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened d8faed9fa602af320f8ab06bb017e46d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65195920, jitterRate=-0.02850508689880371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:21,628 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for d8faed9fa602af320f8ab06bb017e46d: 2024-11-09T09:16:21,629 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d., pid=71, masterSystemTime=1731143781600 2024-11-09T09:16:21,631 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,631 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:16:21,631 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=d8faed9fa602af320f8ab06bb017e46d, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:21,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T09:16:21,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure d8faed9fa602af320f8ab06bb017e46d, server=9b029db41fd9,43449,1731143603787 in 185 msec 2024-11-09T09:16:21,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-09T09:16:21,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=d8faed9fa602af320f8ab06bb017e46d, ASSIGN in 342 msec 2024-11-09T09:16:21,637 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:21,638 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143781637"}]},"ts":"1731143781637"} 2024-11-09T09:16:21,639 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:21,649 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:21,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.2700 sec 2024-11-09T09:16:22,226 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:16:22,746 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001717 expired clientIPAndPort=172.17.0.2:43116, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:16:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-09T09:16:23,392 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-09T09:16:23,392 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:23,393 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:23,402 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:23,402 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:23,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-09T09:16:23,406 DEBUG [hconnection-0x7e6dca0f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:23,407 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48698, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:23,414 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:23,417 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:16:23,417 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:16:23,418 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x09212546 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:16:23,418 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x09212546 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:23,419 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48706, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:16:23,422 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:16:23,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549961 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:23,423 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:16:23,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549961 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:23,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-09T09:16:23,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-09T09:16:23,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:23,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-09T09:16:23,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:26,131 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-09T09:16:26,433 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3010 2024-11-09T09:16:26,434 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4752c8a8 to 127.0.0.1:49767 2024-11-09T09:16:26,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:26,434 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549961 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:16:26,434 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:26,435 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:26,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:26,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001655' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T09:16:26,435 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x09212546 to 127.0.0.1:49767 2024-11-09T09:16:26,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:48706 deadline: 1731143787433, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5312316530116001655' does not exist 2024-11-09T09:16:26,441 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:26,436 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 9b029db41fd9,43449,1731143603787 for closing scanner id = -5312316530116001655 for d8faed9fa602af320f8ab06bb017e46d of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T09:16:26,447 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=230 (was 231), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=17 (was 19), ProcessCount=11 (was 11), AvailableMemoryMB=3385 (was 3413) 2024-11-09T09:16:26,454 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=17, ProcessCount=11, AvailableMemoryMB=3384 2024-11-09T09:16:26,456 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ea1f3cf to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28df07aa 2024-11-09T09:16:26,510 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x6ea1f3cf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4edd6951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:26,511 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f5c87c2 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24b7d583 2024-11-09T09:16:26,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e97551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:26,521 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:26,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-09T09:16:26,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:26,523 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:26,524 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-09T09:16:26,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-09T09:16:26,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741879_1055 (size=416) 2024-11-09T09:16:26,924 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:26,940 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1fbc66753606036aa3753c426d2e6336, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:26,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741880_1056 (size=67) 2024-11-09T09:16:27,113 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:27,354 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:27,354 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1fbc66753606036aa3753c426d2e6336, disabling compactions & flushes 2024-11-09T09:16:27,355 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,355 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,355 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. after waiting 0 ms 2024-11-09T09:16:27,355 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,355 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,355 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1fbc66753606036aa3753c426d2e6336: 2024-11-09T09:16:27,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:27,358 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731143787357"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143787357"}]},"ts":"1731143787357"} 2024-11-09T09:16:27,361 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:27,362 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:27,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143787363"}]},"ts":"1731143787363"} 2024-11-09T09:16:27,365 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:27,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1fbc66753606036aa3753c426d2e6336, ASSIGN}] 2024-11-09T09:16:27,410 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1fbc66753606036aa3753c426d2e6336, ASSIGN 2024-11-09T09:16:27,411 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1fbc66753606036aa3753c426d2e6336, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-09T09:16:27,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=1fbc66753606036aa3753c426d2e6336, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:27,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 1fbc66753606036aa3753c426d2e6336, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:27,720 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:27,732 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,732 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 1fbc66753606036aa3753c426d2e6336, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:27,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,737 INFO [StoreOpener-1fbc66753606036aa3753c426d2e6336-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,739 INFO [StoreOpener-1fbc66753606036aa3753c426d2e6336-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fbc66753606036aa3753c426d2e6336 columnFamilyName testFamily 2024-11-09T09:16:27,739 DEBUG [StoreOpener-1fbc66753606036aa3753c426d2e6336-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:27,740 INFO [StoreOpener-1fbc66753606036aa3753c426d2e6336-1 {}] regionserver.HStore(327): Store=1fbc66753606036aa3753c426d2e6336/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:27,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 1fbc66753606036aa3753c426d2e6336 2024-11-09T09:16:27,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:27,746 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 1fbc66753606036aa3753c426d2e6336; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62483476, jitterRate=-0.06892365217208862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:27,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 1fbc66753606036aa3753c426d2e6336: 2024-11-09T09:16:27,747 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336., pid=74, masterSystemTime=1731143787719 2024-11-09T09:16:27,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,749 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:16:27,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=1fbc66753606036aa3753c426d2e6336, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:27,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-09T09:16:27,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 1fbc66753606036aa3753c426d2e6336, server=9b029db41fd9,43449,1731143603787 in 185 msec 2024-11-09T09:16:27,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T09:16:27,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=1fbc66753606036aa3753c426d2e6336, ASSIGN in 342 msec 2024-11-09T09:16:27,753 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:27,753 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143787753"}]},"ts":"1731143787753"} 2024-11-09T09:16:27,754 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:27,768 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:27,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.2470 sec 2024-11-09T09:16:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-09T09:16:29,532 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-09T09:16:29,532 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:29,532 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:29,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-09T09:16:29,545 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549962 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:29,546 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:29,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:29,546 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-09T09:16:29,548 DEBUG [hconnection-0x610e9a57-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:29,549 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38016, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:29,556 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:29,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:16:29,560 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549965 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:16:29,560 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-09T09:16:29,757 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001715 expired clientIPAndPort=172.17.0.2:43116, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:16:32,135 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-09T09:16:33,237 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-09T09:16:33,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:34,965 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001710 expired clientIPAndPort=172.17.0.2:38864, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:16:35,987 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:16:36,061 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-09T09:16:36,061 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f5c87c2 to 127.0.0.1:49767 2024-11-09T09:16:36,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:36,061 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:36,061 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:36,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:36,061 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ea1f3cf to 127.0.0.1:49767 2024-11-09T09:16:36,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=229 (was 230), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=14 (was 17), ProcessCount=11 (was 11), AvailableMemoryMB=3336 (was 3384) 2024-11-09T09:16:36,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=14, ProcessCount=11, AvailableMemoryMB=3336 2024-11-09T09:16:36,079 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6128809c to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36898f8d 2024-11-09T09:16:36,110 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x6128809c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74984b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:36,112 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2136e770 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30f7e865 2024-11-09T09:16:36,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@157eb287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:36,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-09T09:16:36,134 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:36,134 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:36,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-09T09:16:36,136 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T09:16:36,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741881_1057 (size=420) 2024-11-09T09:16:36,553 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d655bfcbe7771c9343ff43b35183ce3f, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741882_1058 (size=71) 2024-11-09T09:16:36,965 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:36,965 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing d655bfcbe7771c9343ff43b35183ce3f, disabling compactions & flushes 2024-11-09T09:16:36,965 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:36,965 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:36,966 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. after waiting 0 ms 2024-11-09T09:16:36,966 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:36,966 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:36,966 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for d655bfcbe7771c9343ff43b35183ce3f: 2024-11-09T09:16:36,969 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:36,969 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731143796969"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143796969"}]},"ts":"1731143796969"} 2024-11-09T09:16:36,974 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:36,975 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:36,975 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143796975"}]},"ts":"1731143796975"} 2024-11-09T09:16:36,977 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:37,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=d655bfcbe7771c9343ff43b35183ce3f, ASSIGN}] 2024-11-09T09:16:37,037 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=d655bfcbe7771c9343ff43b35183ce3f, ASSIGN 2024-11-09T09:16:37,039 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=d655bfcbe7771c9343ff43b35183ce3f, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T09:16:37,190 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=d655bfcbe7771c9343ff43b35183ce3f, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:37,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure d655bfcbe7771c9343ff43b35183ce3f, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:37,198 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:16:37,346 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:37,358 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:37,358 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => d655bfcbe7771c9343ff43b35183ce3f, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:37,358 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,358 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:37,359 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,359 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,360 INFO [StoreOpener-d655bfcbe7771c9343ff43b35183ce3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,362 INFO [StoreOpener-d655bfcbe7771c9343ff43b35183ce3f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d655bfcbe7771c9343ff43b35183ce3f columnFamilyName testFamily 2024-11-09T09:16:37,362 DEBUG [StoreOpener-d655bfcbe7771c9343ff43b35183ce3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:37,363 INFO [StoreOpener-d655bfcbe7771c9343ff43b35183ce3f-1 {}] regionserver.HStore(327): Store=d655bfcbe7771c9343ff43b35183ce3f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:37,364 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,364 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,367 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:16:37,370 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:37,370 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened d655bfcbe7771c9343ff43b35183ce3f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65686063, jitterRate=-0.02120138704776764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:37,371 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for d655bfcbe7771c9343ff43b35183ce3f: 2024-11-09T09:16:37,372 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f., pid=77, masterSystemTime=1731143797346 2024-11-09T09:16:37,374 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:37,374 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:16:37,375 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=d655bfcbe7771c9343ff43b35183ce3f, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:37,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T09:16:37,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure d655bfcbe7771c9343ff43b35183ce3f, server=9b029db41fd9,43449,1731143603787 in 183 msec 2024-11-09T09:16:37,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-09T09:16:37,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=d655bfcbe7771c9343ff43b35183ce3f, ASSIGN in 344 msec 2024-11-09T09:16:37,380 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:37,381 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143797380"}]},"ts":"1731143797380"} 2024-11-09T09:16:37,382 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:37,393 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:37,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.2630 sec 2024-11-09T09:16:38,139 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-09T09:16:39,071 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001705 expired clientIPAndPort=172.17.0.2:38892, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:16:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T09:16:39,143 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-09T09:16:39,143 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:39,144 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:39,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-09T09:16:39,153 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549966 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:39,155 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:39,155 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:39,155 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-09T09:16:39,158 DEBUG [hconnection-0x745a93cf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:39,159 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40390, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:39,165 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:39,167 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:16:39,168 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549969 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T09:16:39,168 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-09T09:16:40,670 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-09T09:16:40,671 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2136e770 to 127.0.0.1:49767 2024-11-09T09:16:40,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:40,671 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:40,672 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:40,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:40,673 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6128809c to 127.0.0.1:49767 2024-11-09T09:16:40,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 229) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=13 (was 14), ProcessCount=11 (was 11), AvailableMemoryMB=3325 (was 3336) 2024-11-09T09:16:40,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=13, ProcessCount=11, AvailableMemoryMB=3325 2024-11-09T09:16:40,700 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x790dd9a6 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c1074df 2024-11-09T09:16:40,718 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x790dd9a6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca0d5a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:40,719 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a230143 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@588d95c1 2024-11-09T09:16:40,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5ab79f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:40,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-09T09:16:40,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:40,730 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:40,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-09T09:16:40,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-09T09:16:40,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741883_1059 (size=419) 2024-11-09T09:16:41,079 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001703 expired clientIPAndPort=172.17.0.2:38892, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:16:41,141 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 716f08f80db337264acc1e40d08de6fb, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:41,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741884_1060 (size=70) 2024-11-09T09:16:41,555 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:41,555 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 716f08f80db337264acc1e40d08de6fb, disabling compactions & flushes 2024-11-09T09:16:41,555 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,555 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,555 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. after waiting 0 ms 2024-11-09T09:16:41,556 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,556 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,556 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 716f08f80db337264acc1e40d08de6fb: 2024-11-09T09:16:41,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:41,559 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731143801559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143801559"}]},"ts":"1731143801559"} 2024-11-09T09:16:41,562 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:41,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:41,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143801563"}]},"ts":"1731143801563"} 2024-11-09T09:16:41,565 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:41,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=716f08f80db337264acc1e40d08de6fb, ASSIGN}] 2024-11-09T09:16:41,611 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=716f08f80db337264acc1e40d08de6fb, ASSIGN 2024-11-09T09:16:41,612 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=716f08f80db337264acc1e40d08de6fb, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-09T09:16:41,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=716f08f80db337264acc1e40d08de6fb, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:41,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 716f08f80db337264acc1e40d08de6fb, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:41,921 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:41,928 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,929 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => 716f08f80db337264acc1e40d08de6fb, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:41,929 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,930 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:41,930 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for 716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,930 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for 716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,933 INFO [StoreOpener-716f08f80db337264acc1e40d08de6fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,936 INFO [StoreOpener-716f08f80db337264acc1e40d08de6fb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 716f08f80db337264acc1e40d08de6fb columnFamilyName testFamily 2024-11-09T09:16:41,937 DEBUG [StoreOpener-716f08f80db337264acc1e40d08de6fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:41,937 INFO [StoreOpener-716f08f80db337264acc1e40d08de6fb-1 {}] regionserver.HStore(327): Store=716f08f80db337264acc1e40d08de6fb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:41,939 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,939 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,943 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for 716f08f80db337264acc1e40d08de6fb 2024-11-09T09:16:41,946 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:41,947 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened 716f08f80db337264acc1e40d08de6fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67530136, jitterRate=0.00627744197845459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:41,948 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for 716f08f80db337264acc1e40d08de6fb: 2024-11-09T09:16:41,949 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb., pid=80, masterSystemTime=1731143801921 2024-11-09T09:16:41,951 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,951 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:16:41,951 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=716f08f80db337264acc1e40d08de6fb, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:41,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-09T09:16:41,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 716f08f80db337264acc1e40d08de6fb, server=9b029db41fd9,43449,1731143603787 in 185 msec 2024-11-09T09:16:41,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T09:16:41,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=716f08f80db337264acc1e40d08de6fb, ASSIGN in 346 msec 2024-11-09T09:16:41,957 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:41,957 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143801957"}]},"ts":"1731143801957"} 2024-11-09T09:16:41,958 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:42,000 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:42,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.2720 sec 2024-11-09T09:16:42,863 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:43,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-09T09:16:43,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:43,449 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-09T09:16:43,449 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-09T09:16:43,740 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-09T09:16:43,740 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:43,741 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:43,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-09T09:16:43,746 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549970 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:43,748 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:43,748 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:43,748 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-09T09:16:43,750 DEBUG [hconnection-0x600da46a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:43,751 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:43,758 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:43,760 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:16:43,762 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:16:43,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549973 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:43,763 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:16:43,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549973 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:43,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-09T09:16:44,143 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-09T09:16:46,773 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4011: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3008ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4011: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3008ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3008ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3008ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:16:46,774 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3010 2024-11-09T09:16:46,775 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a230143 to 127.0.0.1:49767 2024-11-09T09:16:46,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:46,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:46,776 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:46,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:46,777 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x790dd9a6 to 127.0.0.1:49767 2024-11-09T09:16:46,790 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=11 (was 13), ProcessCount=11 (was 11), AvailableMemoryMB=3246 (was 3325) 2024-11-09T09:16:46,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=11, ProcessCount=11, AvailableMemoryMB=3245 2024-11-09T09:16:46,800 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f7c5ce3 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f52d7eb 2024-11-09T09:16:46,818 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x0f7c5ce3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef62297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:46,819 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ae7d28d to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7940d4c2 2024-11-09T09:16:46,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1436044d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:46,835 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:46,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-09T09:16:46,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:46,837 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:46,837 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-09T09:16:46,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T09:16:46,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741885_1061 (size=421) 2024-11-09T09:16:47,249 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 999c6baaaa000b15d3dbb1d6dc21b2c4, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:47,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741886_1062 (size=72) 2024-11-09T09:16:47,265 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:16:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 999c6baaaa000b15d3dbb1d6dc21b2c4, disabling compactions & flushes 2024-11-09T09:16:47,663 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. after waiting 0 ms 2024-11-09T09:16:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:47,663 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 999c6baaaa000b15d3dbb1d6dc21b2c4: 2024-11-09T09:16:47,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:47,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731143807665"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143807665"}]},"ts":"1731143807665"} 2024-11-09T09:16:47,668 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:47,669 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:47,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143807670"}]},"ts":"1731143807670"} 2024-11-09T09:16:47,672 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:47,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=999c6baaaa000b15d3dbb1d6dc21b2c4, ASSIGN}] 2024-11-09T09:16:47,719 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=999c6baaaa000b15d3dbb1d6dc21b2c4, ASSIGN 2024-11-09T09:16:47,721 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=999c6baaaa000b15d3dbb1d6dc21b2c4, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T09:16:47,871 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=999c6baaaa000b15d3dbb1d6dc21b2c4, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:47,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 999c6baaaa000b15d3dbb1d6dc21b2c4, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:48,025 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:48,081 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:48,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => 999c6baaaa000b15d3dbb1d6dc21b2c4, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:48,082 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,082 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:48,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for 999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for 999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,087 INFO [StoreOpener-999c6baaaa000b15d3dbb1d6dc21b2c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,090 INFO [StoreOpener-999c6baaaa000b15d3dbb1d6dc21b2c4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999c6baaaa000b15d3dbb1d6dc21b2c4 columnFamilyName testFamily 2024-11-09T09:16:48,090 DEBUG [StoreOpener-999c6baaaa000b15d3dbb1d6dc21b2c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:48,091 INFO [StoreOpener-999c6baaaa000b15d3dbb1d6dc21b2c4-1 {}] regionserver.HStore(327): Store=999c6baaaa000b15d3dbb1d6dc21b2c4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:48,092 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for 999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:16:48,098 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:48,098 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened 999c6baaaa000b15d3dbb1d6dc21b2c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72019455, jitterRate=0.07317350804805756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:48,099 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for 999c6baaaa000b15d3dbb1d6dc21b2c4: 2024-11-09T09:16:48,100 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4., pid=83, masterSystemTime=1731143808025 2024-11-09T09:16:48,101 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:48,101 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:16:48,102 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=999c6baaaa000b15d3dbb1d6dc21b2c4, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:48,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T09:16:48,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 999c6baaaa000b15d3dbb1d6dc21b2c4, server=9b029db41fd9,43449,1731143603787 in 230 msec 2024-11-09T09:16:48,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-09T09:16:48,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=999c6baaaa000b15d3dbb1d6dc21b2c4, ASSIGN in 387 msec 2024-11-09T09:16:48,106 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:48,106 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143808106"}]},"ts":"1731143808106"} 2024-11-09T09:16:48,107 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:48,177 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:48,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.3430 sec 2024-11-09T09:16:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T09:16:49,841 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-09T09:16:49,841 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:49,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:49,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-09T09:16:49,845 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549974 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:49,847 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:49,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:49,847 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-09T09:16:49,849 DEBUG [hconnection-0x4c05f804-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:49,851 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33830, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:49,858 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:49,860 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:16:49,862 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:16:49,862 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-09T09:16:50,146 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-09T09:16:51,294 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001689 expired clientIPAndPort=172.17.0.2:57300, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:16:52,227 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:16:53,627 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:16:53,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-09T09:16:53,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:16:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T09:16:56,884 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:16:56,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:33830 deadline: 1731143818861, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T09:16:56,886 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T09:16:56,887 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13134427543593549977 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T09:16:56,887 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7027 2024-11-09T09:16:56,888 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3ae7d28d to 127.0.0.1:49767 2024-11-09T09:16:56,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:56,888 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:16:56,888 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:16:56,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:16:56,888 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f7c5ce3 to 127.0.0.1:49767 2024-11-09T09:16:56,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=24 (was 11) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3239 (was 3245) 2024-11-09T09:16:56,909 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=24, ProcessCount=11, AvailableMemoryMB=3238 2024-11-09T09:16:56,911 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71e35a50 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d6bee89 2024-11-09T09:16:56,942 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x71e35a50 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@584b1d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:56,943 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ec3fbd2 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f92cd61 2024-11-09T09:16:56,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74482519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:16:56,952 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:16:56,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T09:16:56,954 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:16:56,954 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:56,954 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-09T09:16:56,955 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:16:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-09T09:16:56,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741887_1063 (size=427) 2024-11-09T09:16:57,370 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => abf768636ae7ebb0ba6696eb118fa2ba, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:16:57,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741888_1064 (size=78) 2024-11-09T09:16:57,781 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:57,781 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing abf768636ae7ebb0ba6696eb118fa2ba, disabling compactions & flushes 2024-11-09T09:16:57,781 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:57,781 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:57,781 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. after waiting 0 ms 2024-11-09T09:16:57,782 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:57,782 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:57,782 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for abf768636ae7ebb0ba6696eb118fa2ba: 2024-11-09T09:16:57,784 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:16:57,785 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731143817785"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143817785"}]},"ts":"1731143817785"} 2024-11-09T09:16:57,788 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:16:57,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:16:57,789 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143817789"}]},"ts":"1731143817789"} 2024-11-09T09:16:57,791 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:16:57,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=abf768636ae7ebb0ba6696eb118fa2ba, ASSIGN}] 2024-11-09T09:16:57,834 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=abf768636ae7ebb0ba6696eb118fa2ba, ASSIGN 2024-11-09T09:16:57,835 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=abf768636ae7ebb0ba6696eb118fa2ba, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:16:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-09T09:16:57,986 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=abf768636ae7ebb0ba6696eb118fa2ba, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:57,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure abf768636ae7ebb0ba6696eb118fa2ba, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:16:58,141 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:16:58,151 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:58,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => abf768636ae7ebb0ba6696eb118fa2ba, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:16:58,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:16:58,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,154 INFO [StoreOpener-abf768636ae7ebb0ba6696eb118fa2ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,156 INFO [StoreOpener-abf768636ae7ebb0ba6696eb118fa2ba-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abf768636ae7ebb0ba6696eb118fa2ba columnFamilyName testFamily 2024-11-09T09:16:58,156 DEBUG [StoreOpener-abf768636ae7ebb0ba6696eb118fa2ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:16:58,157 INFO [StoreOpener-abf768636ae7ebb0ba6696eb118fa2ba-1 {}] regionserver.HStore(327): Store=abf768636ae7ebb0ba6696eb118fa2ba/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:16:58,158 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,158 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,161 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:16:58,163 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:16:58,164 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened abf768636ae7ebb0ba6696eb118fa2ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70806199, jitterRate=0.055094584822654724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:16:58,164 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for abf768636ae7ebb0ba6696eb118fa2ba: 2024-11-09T09:16:58,165 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., pid=86, masterSystemTime=1731143818141 2024-11-09T09:16:58,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:58,167 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:16:58,167 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=abf768636ae7ebb0ba6696eb118fa2ba, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:16:58,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-09T09:16:58,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure abf768636ae7ebb0ba6696eb118fa2ba, server=9b029db41fd9,43449,1731143603787 in 180 msec 2024-11-09T09:16:58,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T09:16:58,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=abf768636ae7ebb0ba6696eb118fa2ba, ASSIGN in 337 msec 2024-11-09T09:16:58,171 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:16:58,172 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143818171"}]},"ts":"1731143818171"} 2024-11-09T09:16:58,173 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:16:58,184 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:16:58,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.2310 sec 2024-11-09T09:16:58,304 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001687 expired clientIPAndPort=172.17.0.2:57300, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:16:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-09T09:16:59,962 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-09T09:16:59,962 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:16:59,963 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:59,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-09T09:16:59,972 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549979 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:16:59,973 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T09:16:59,973 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:16:59,973 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-09T09:16:59,975 DEBUG [hconnection-0x71866516-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:59,976 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44700, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:59,982 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:16:59,984 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:16:59,984 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:16:59,985 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x71e35a50 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:16:59,986 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x71e35a50 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:16:59,987 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44704, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:16:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:16:59,989 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:17:02,151 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-09T09:17:03,656 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:17:03,700 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T09:17:03,700 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:17:05,993 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2024-11-09T09:17:05,993 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2024-11-09T09:17:05,993 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:17:06,144 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:17:06,416 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001680 expired clientIPAndPort=172.17.0.2:49752, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:17:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T09:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:17:07,007 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T09:17:07,373 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:17:08,992 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms 2024-11-09T09:17:08,993 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms 2024-11-09T09:17:08,993 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:17:08,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T09:17:05.993Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2024-11-09T09:17:08.993Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T09:17:08,997 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9013 2024-11-09T09:17:08,998 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ec3fbd2 to 127.0.0.1:49767 2024-11-09T09:17:08,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:08,998 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:17:08,999 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:17:08,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:08,999 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x71e35a50 to 127.0.0.1:49767 2024-11-09T09:17:09,009 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=225 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=28 (was 24) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3224 (was 3238) 2024-11-09T09:17:09,018 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=225, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=28, ProcessCount=11, AvailableMemoryMB=3223 2024-11-09T09:17:09,019 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ebf2942 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@649527ac 2024-11-09T09:17:09,020 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001676 expired clientIPAndPort=172.17.0.2:59194, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:09,063 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7ebf2942 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f240aa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:17:09,066 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61e2dde8 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@746032f7 2024-11-09T09:17:09,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7312e000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:17:09,079 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:17:09,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-09T09:17:09,081 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:17:09,081 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:17:09,081 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-09T09:17:09,081 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:17:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T09:17:09,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741889_1065 (size=415) 2024-11-09T09:17:09,120 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001674 expired clientIPAndPort=172.17.0.2:59194, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:09,120 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001673 expired clientIPAndPort=172.17.0.2:59194, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:09,497 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0699f50861d7de54ae4352ede8ba637f, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:17:09,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741890_1066 (size=66) 2024-11-09T09:17:09,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:17:09,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 0699f50861d7de54ae4352ede8ba637f, disabling compactions & flushes 2024-11-09T09:17:09,912 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:09,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:09,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. after waiting 0 ms 2024-11-09T09:17:09,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:09,912 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:09,913 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0699f50861d7de54ae4352ede8ba637f: 2024-11-09T09:17:09,913 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:17:09,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731143829914"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143829914"}]},"ts":"1731143829914"} 2024-11-09T09:17:09,915 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:17:09,916 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:17:09,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143829916"}]},"ts":"1731143829916"} 2024-11-09T09:17:09,918 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:17:09,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=0699f50861d7de54ae4352ede8ba637f, ASSIGN}] 2024-11-09T09:17:09,967 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=0699f50861d7de54ae4352ede8ba637f, ASSIGN 2024-11-09T09:17:09,967 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=0699f50861d7de54ae4352ede8ba637f, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:17:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T09:17:10,118 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=0699f50861d7de54ae4352ede8ba637f, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:17:10,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure 0699f50861d7de54ae4352ede8ba637f, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:17:10,277 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:17:10,291 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:10,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => 0699f50861d7de54ae4352ede8ba637f, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:17:10,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true 0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:17:10,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for 0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for 0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,293 INFO [StoreOpener-0699f50861d7de54ae4352ede8ba637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,295 INFO [StoreOpener-0699f50861d7de54ae4352ede8ba637f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0699f50861d7de54ae4352ede8ba637f columnFamilyName testFamily 2024-11-09T09:17:10,295 DEBUG [StoreOpener-0699f50861d7de54ae4352ede8ba637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:17:10,295 INFO [StoreOpener-0699f50861d7de54ae4352ede8ba637f-1 {}] regionserver.HStore(327): Store=0699f50861d7de54ae4352ede8ba637f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:17:10,296 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,297 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for 0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:10,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:17:10,302 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened 0699f50861d7de54ae4352ede8ba637f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72174955, jitterRate=0.07549063861370087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:17:10,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for 0699f50861d7de54ae4352ede8ba637f: 2024-11-09T09:17:10,303 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f., pid=89, masterSystemTime=1731143830277 2024-11-09T09:17:10,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:10,305 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:10,305 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=0699f50861d7de54ae4352ede8ba637f, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:17:10,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-09T09:17:10,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure 0699f50861d7de54ae4352ede8ba637f, server=9b029db41fd9,43449,1731143603787 in 185 msec 2024-11-09T09:17:10,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-09T09:17:10,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=0699f50861d7de54ae4352ede8ba637f, ASSIGN in 343 msec 2024-11-09T09:17:10,311 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:17:10,311 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143830311"}]},"ts":"1731143830311"} 2024-11-09T09:17:10,313 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:17:10,350 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:17:10,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.2710 sec 2024-11-09T09:17:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T09:17:12,088 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-09T09:17:12,088 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:17:12,089 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:17:12,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-09T09:17:12,100 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549986 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:17:12,103 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-09T09:17:12,103 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:17:12,103 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-09T09:17:12,105 DEBUG [hconnection-0x5e04bed0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:17:12,106 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41688, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:17:12,113 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:17:12,115 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T09:17:12,116 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T09:17:12,116 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549989 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:17:12,117 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T09:17:12,117 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549989 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:17:12,118 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-09T09:17:13,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-09T09:17:13,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:17:13,507 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:17:14,155 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-09T09:17:14,227 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001667 expired clientIPAndPort=172.17.0.2:59210, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:17:15,838 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:17:18,834 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001662 expired clientIPAndPort=172.17.0.2:59212, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:20,838 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001660 expired clientIPAndPort=172.17.0.2:59212, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:21,123 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10007: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9005ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10007: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9005ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9005ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9005ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T09:17:21,123 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9006 2024-11-09T09:17:21,124 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61e2dde8 to 127.0.0.1:49767 2024-11-09T09:17:21,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:21,124 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:17:21,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:17:21,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:21,125 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ebf2942 to 127.0.0.1:49767 2024-11-09T09:17:21,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226 (was 225) - Thread LEAK? -, OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=23 (was 28), ProcessCount=11 (was 11), AvailableMemoryMB=3213 (was 3223) 2024-11-09T09:17:21,150 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=23, ProcessCount=11, AvailableMemoryMB=3213 2024-11-09T09:17:21,152 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ed81d49 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47557f20 2024-11-09T09:17:21,166 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7ed81d49 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fed2c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:17:21,167 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d0f7c29 to 127.0.0.1:49767 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d99cbb0 2024-11-09T09:17:21,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7395a51f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T09:17:21,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T09:17:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T09:17:21,181 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T09:17:21,181 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:17:21,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-09T09:17:21,182 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T09:17:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-09T09:17:21,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741891_1067 (size=431) 2024-11-09T09:17:21,599 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9052953756601a7a9f875737f91f1fa6, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014 2024-11-09T09:17:21,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741892_1068 (size=82) 2024-11-09T09:17:21,618 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:17:22,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:17:22,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 9052953756601a7a9f875737f91f1fa6, disabling compactions & flushes 2024-11-09T09:17:22,010 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. after waiting 0 ms 2024-11-09T09:17:22,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,010 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9052953756601a7a9f875737f91f1fa6: 2024-11-09T09:17:22,011 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T09:17:22,011 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731143842011"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731143842011"}]},"ts":"1731143842011"} 2024-11-09T09:17:22,013 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T09:17:22,014 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T09:17:22,014 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143842014"}]},"ts":"1731143842014"} 2024-11-09T09:17:22,015 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T09:17:22,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=9052953756601a7a9f875737f91f1fa6, ASSIGN}] 2024-11-09T09:17:22,059 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=9052953756601a7a9f875737f91f1fa6, ASSIGN 2024-11-09T09:17:22,061 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=9052953756601a7a9f875737f91f1fa6, ASSIGN; state=OFFLINE, location=9b029db41fd9,43449,1731143603787; forceNewPlan=false, retain=false 2024-11-09T09:17:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-09T09:17:22,212 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=9052953756601a7a9f875737f91f1fa6, regionState=OPENING, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:17:22,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 9052953756601a7a9f875737f91f1fa6, server=9b029db41fd9,43449,1731143603787}] 2024-11-09T09:17:22,227 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T09:17:22,370 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9b029db41fd9,43449,1731143603787 2024-11-09T09:17:22,379 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,379 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 9052953756601a7a9f875737f91f1fa6, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6.', STARTKEY => '', ENDKEY => ''} 2024-11-09T09:17:22,380 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,380 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T09:17:22,380 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,381 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,384 INFO [StoreOpener-9052953756601a7a9f875737f91f1fa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,386 INFO [StoreOpener-9052953756601a7a9f875737f91f1fa6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9052953756601a7a9f875737f91f1fa6 columnFamilyName testFamily 2024-11-09T09:17:22,386 DEBUG [StoreOpener-9052953756601a7a9f875737f91f1fa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T09:17:22,387 INFO [StoreOpener-9052953756601a7a9f875737f91f1fa6-1 {}] regionserver.HStore(327): Store=9052953756601a7a9f875737f91f1fa6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T09:17:22,388 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,389 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,392 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:22,395 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T09:17:22,395 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 9052953756601a7a9f875737f91f1fa6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72369987, jitterRate=0.07839684188365936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T09:17:22,396 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 9052953756601a7a9f875737f91f1fa6: 2024-11-09T09:17:22,397 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., pid=92, masterSystemTime=1731143842370 2024-11-09T09:17:22,399 DEBUG [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,399 INFO [RS_OPEN_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:22,400 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=9052953756601a7a9f875737f91f1fa6, regionState=OPEN, openSeqNum=2, regionLocation=9b029db41fd9,43449,1731143603787 2024-11-09T09:17:22,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T09:17:22,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 9052953756601a7a9f875737f91f1fa6, server=9b029db41fd9,43449,1731143603787 in 186 msec 2024-11-09T09:17:22,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-09T09:17:22,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=9052953756601a7a9f875737f91f1fa6, ASSIGN in 347 msec 2024-11-09T09:17:22,407 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T09:17:22,407 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731143842407"}]},"ts":"1731143842407"} 2024-11-09T09:17:22,408 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T09:17:22,441 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T09:17:22,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.2630 sec 2024-11-09T09:17:23,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T09:17:23,447 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T09:17:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41847 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-09T09:17:24,190 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-09T09:17:24,190 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T09:17:24,190 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:17:24,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43449 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-09T09:17:24,200 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13134427543593549990 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T09:17:24,201 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T09:17:24,201 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T09:17:24,201 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-09T09:17:24,203 DEBUG [hconnection-0x6029285f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:17:24,204 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49538, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:17:24,211 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T09:17:24,212 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T09:17:24,213 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T09:17:24,214 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7ed81d49 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9b029db41fd9,43449,1731143603787, seqNum=-1] 2024-11-09T09:17:24,214 DEBUG [ReadOnlyZKClient-127.0.0.1:49767@0x7ed81d49 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T09:17:24,215 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49540, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T09:17:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:17:24,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:17:25,223 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T09:17:25,223 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T09:17:25,223 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:17:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T09:17:26,159 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-09T09:17:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2] 2024-11-09T09:17:26,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43449 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T09:17:26,946 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5312316530116001655 expired clientIPAndPort=172.17.0.2:48706, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:17:27,222 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=967ms 2024-11-09T09:17:27,222 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=967ms 2024-11-09T09:17:27,222 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6., hostname=9b029db41fd9,43449,1731143603787, seqNum=2 from cache 2024-11-09T09:17:27,223 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T09:17:25.223Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T09:17:27.222Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=967ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=9b029db41fd9:43449 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=967ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=967ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T09:17:27,225 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3012 2024-11-09T09:17:27,225 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d0f7c29 to 127.0.0.1:49767 2024-11-09T09:17:27,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:27,225 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T09:17:27,225 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T09:17:27,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:27,226 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ed81d49 to 127.0.0.1:49767 2024-11-09T09:17:27,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=229 (was 226) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=27 (was 23) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3206 (was 3213) 2024-11-09T09:17:27,237 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-09T09:17:27,237 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T09:17:27,237 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x799a723d to 127.0.0.1:49767 2024-11-09T09:17:27,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:27,237 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T09:17:27,237 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=172249665, stopped=false 2024-11-09T09:17:27,238 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=9b029db41fd9,41847,1731143603026 2024-11-09T09:17:27,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T09:17:27,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T09:17:27,249 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-09T09:17:27,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:17:27,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:17:27,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:27,249 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '9b029db41fd9,43449,1731143603787' ***** 2024-11-09T09:17:27,249 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-09T09:17:27,250 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T09:17:27,250 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T09:17:27,250 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T09:17:27,250 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-09T09:17:27,251 INFO [RS:0;9b029db41fd9:43449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T09:17:27,251 INFO [RS:0;9b029db41fd9:43449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T09:17:27,251 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for d655bfcbe7771c9343ff43b35183ce3f 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 716f08f80db337264acc1e40d08de6fb 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 4622f677b70a31e1a9ad71877e78cdbc 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 00c12a1ffd145dac3402d64c48cc20c7 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 1fbc66753606036aa3753c426d2e6336 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 2ed876c86d7febba0ebb3221271b6b23 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 3a971eb4fc571ad608de182c691878da 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 0699f50861d7de54ae4352ede8ba637f 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 271a1f8adda9f40e39312afa694ac217 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for ca546e27319aa177954e402fd053bd2b 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 316c4850c2b79d7cbe87367fe6cd94b3 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 496f9744c3b0aceac24ffc23bd44774f 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 999c6baaaa000b15d3dbb1d6dc21b2c4 2024-11-09T09:17:27,252 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d655bfcbe7771c9343ff43b35183ce3f, disabling compactions & flushes 2024-11-09T09:17:27,252 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 9b80974f1b4163166250b30045e5a88b 2024-11-09T09:17:27,253 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for d8faed9fa602af320f8ab06bb017e46d 2024-11-09T09:17:27,253 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 36ec248a87a5f6a35a3be224ec1de4c5 2024-11-09T09:17:27,253 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. after waiting 0 ms 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for c4fd6ba823547d504f5d98c1c026399e 2024-11-09T09:17:27,253 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 9052953756601a7a9f875737f91f1fa6 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for ef34766cd3a29a746f1b7db5199d51e4 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for f6ef5e815e8a0538da916d41f9c6bba5 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for d29da8fc7e2c0292000b4931ecdf1ed1 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for ba96f0895981853116801be5af09f3d2 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 459f96eb1d922bcd9f696d43a6691e8c 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for cc68045eb729be737b2bb40a8dcab3f4 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:17:27,253 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d655bfcbe7771c9343ff43b35183ce3f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 46c966491675fb74bc449a58cad47886 2024-11-09T09:17:27,253 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:17:27,254 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1224): stopping server 9b029db41fd9,43449,1731143603787 2024-11-09T09:17:27,254 DEBUG [RS:0;9b029db41fd9:43449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:27,254 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T09:17:27,254 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T09:17:27,254 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T09:17:27,254 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-09T09:17:27,254 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-09T09:17:27,255 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-09T09:17:27,255 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-09T09:17:27,255 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-09T09:17:27,255 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T09:17:27,255 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T09:17:27,255 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-09T09:17:27,255 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1588230740, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 716f08f80db337264acc1e40d08de6fb, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d655bfcbe7771c9343ff43b35183ce3f, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:27,310 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/.tmp/testFamily/4449941de7f94264ad20c11324e7c33e is 61, key is row-0/testFamily:testQualifier/1731143799161/Put/seqid=0 2024-11-09T09:17:27,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741893_1069 (size=5049) 2024-11-09T09:17:27,322 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/.tmp/info/c72d89db81974f868cf07baab021ec49 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c./info:regioninfo/1731143639668/Put/seqid=0 2024-11-09T09:17:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741894_1070 (size=51873) 2024-11-09T09:17:27,456 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1588230740, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 716f08f80db337264acc1e40d08de6fb, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d655bfcbe7771c9343ff43b35183ce3f, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:27,656 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1588230740, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 716f08f80db337264acc1e40d08de6fb, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d655bfcbe7771c9343ff43b35183ce3f, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:27,699 INFO [regionserver/9b029db41fd9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T09:17:27,699 INFO [regionserver/9b029db41fd9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T09:17:27,722 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/.tmp/testFamily/4449941de7f94264ad20c11324e7c33e 2024-11-09T09:17:27,728 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/.tmp/info/c72d89db81974f868cf07baab021ec49 2024-11-09T09:17:27,748 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T09:17:27,767 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/.tmp/testFamily/4449941de7f94264ad20c11324e7c33e as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/testFamily/4449941de7f94264ad20c11324e7c33e 2024-11-09T09:17:27,774 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/testFamily/4449941de7f94264ad20c11324e7c33e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:27,776 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d655bfcbe7771c9343ff43b35183ce3f in 523ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:27,781 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_1_-true/d655bfcbe7771c9343ff43b35183ce3f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:27,783 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:17:27,783 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d655bfcbe7771c9343ff43b35183ce3f: 2024-11-09T09:17:27,783 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731143796129.d655bfcbe7771c9343ff43b35183ce3f. 2024-11-09T09:17:27,784 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 716f08f80db337264acc1e40d08de6fb, disabling compactions & flushes 2024-11-09T09:17:27,784 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:17:27,784 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:17:27,784 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. after waiting 0 ms 2024-11-09T09:17:27,784 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:17:27,784 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 716f08f80db337264acc1e40d08de6fb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:27,785 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/.tmp/table/f93b0206064645bf8a9adba8c4d68e73 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731143639680/Put/seqid=0 2024-11-09T09:17:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741895_1071 (size=9821) 2024-11-09T09:17:27,800 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/.tmp/testFamily/ce054125bd3c4f27a7b28f030fd767ae is 61, key is row-0/testFamily:testQualifier/1731143803753/Put/seqid=0 2024-11-09T09:17:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741896_1072 (size=5049) 2024-11-09T09:17:27,857 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1588230740, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 716f08f80db337264acc1e40d08de6fb, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:27,924 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:17:28,057 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1588230740, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 716f08f80db337264acc1e40d08de6fb, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:28,191 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/.tmp/table/f93b0206064645bf8a9adba8c4d68e73 2024-11-09T09:17:28,205 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/.tmp/testFamily/ce054125bd3c4f27a7b28f030fd767ae 2024-11-09T09:17:28,208 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/.tmp/info/c72d89db81974f868cf07baab021ec49 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/info/c72d89db81974f868cf07baab021ec49 2024-11-09T09:17:28,213 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/.tmp/testFamily/ce054125bd3c4f27a7b28f030fd767ae as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/testFamily/ce054125bd3c4f27a7b28f030fd767ae 2024-11-09T09:17:28,215 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/info/c72d89db81974f868cf07baab021ec49, entries=290, sequenceid=149, filesize=50.7 K 2024-11-09T09:17:28,216 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/.tmp/table/f93b0206064645bf8a9adba8c4d68e73 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/table/f93b0206064645bf8a9adba8c4d68e73 2024-11-09T09:17:28,220 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/testFamily/ce054125bd3c4f27a7b28f030fd767ae, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:28,222 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/table/f93b0206064645bf8a9adba8c4d68e73, entries=58, sequenceid=149, filesize=9.6 K 2024-11-09T09:17:28,223 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 716f08f80db337264acc1e40d08de6fb in 439ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:28,223 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 968ms, sequenceid=149, compaction requested=false 2024-11-09T09:17:28,227 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-09T09:17:28,227 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_1_-true/716f08f80db337264acc1e40d08de6fb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:28,227 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:17:28,227 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T09:17:28,227 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 716f08f80db337264acc1e40d08de6fb: 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731143800728.716f08f80db337264acc1e40d08de6fb. 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4622f677b70a31e1a9ad71877e78cdbc, disabling compactions & flushes 2024-11-09T09:17:28,228 INFO [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-09T09:17:28,228 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_META-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. after waiting 0 ms 2024-11-09T09:17:28,228 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:28,228 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4622f677b70a31e1a9ad71877e78cdbc 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:28,242 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/.tmp/testFamily/bd339da2e1cd4754a0a09dc9acdfb729 is 61, key is row-0/testFamily:testQualifier/1731143777251/Put/seqid=0 2024-11-09T09:17:28,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741897_1073 (size=5049) 2024-11-09T09:17:28,257 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-09T09:17:28,258 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:28,458 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:28,649 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/.tmp/testFamily/bd339da2e1cd4754a0a09dc9acdfb729 2024-11-09T09:17:28,659 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 4622f677b70a31e1a9ad71877e78cdbc, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:28,666 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/.tmp/testFamily/bd339da2e1cd4754a0a09dc9acdfb729 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/testFamily/bd339da2e1cd4754a0a09dc9acdfb729 2024-11-09T09:17:28,675 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/testFamily/bd339da2e1cd4754a0a09dc9acdfb729, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:28,676 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4622f677b70a31e1a9ad71877e78cdbc in 448ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:28,679 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/4622f677b70a31e1a9ad71877e78cdbc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:28,680 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:28,680 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4622f677b70a31e1a9ad71877e78cdbc: 2024-11-09T09:17:28,680 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731143774224.4622f677b70a31e1a9ad71877e78cdbc. 2024-11-09T09:17:28,680 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 00c12a1ffd145dac3402d64c48cc20c7, disabling compactions & flushes 2024-11-09T09:17:28,680 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:17:28,680 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:17:28,680 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. after waiting 0 ms 2024-11-09T09:17:28,680 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:17:28,680 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 00c12a1ffd145dac3402d64c48cc20c7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:28,695 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/.tmp/testFamily/c2bc77faad084468ad1fac533aaca6e6 is 61, key is row-0/testFamily:testQualifier/1731143687600/Put/seqid=0 2024-11-09T09:17:28,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741898_1074 (size=5049) 2024-11-09T09:17:28,859 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:29,060 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 00c12a1ffd145dac3402d64c48cc20c7, 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:29,101 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/.tmp/testFamily/c2bc77faad084468ad1fac533aaca6e6 2024-11-09T09:17:29,117 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/.tmp/testFamily/c2bc77faad084468ad1fac533aaca6e6 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/testFamily/c2bc77faad084468ad1fac533aaca6e6 2024-11-09T09:17:29,125 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/testFamily/c2bc77faad084468ad1fac533aaca6e6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:29,126 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 00c12a1ffd145dac3402d64c48cc20c7 in 446ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:29,131 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_0_-false/00c12a1ffd145dac3402d64c48cc20c7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:29,132 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:17:29,132 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 00c12a1ffd145dac3402d64c48cc20c7: 2024-11-09T09:17:29,132 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731143684568.00c12a1ffd145dac3402d64c48cc20c7. 2024-11-09T09:17:29,132 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1fbc66753606036aa3753c426d2e6336, disabling compactions & flushes 2024-11-09T09:17:29,132 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:17:29,132 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:17:29,133 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. after waiting 0 ms 2024-11-09T09:17:29,133 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:17:29,133 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1fbc66753606036aa3753c426d2e6336 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:29,151 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/.tmp/testFamily/b4137d7ae6114c87bb415ebd37d74e40 is 61, key is row-0/testFamily:testQualifier/1731143789551/Put/seqid=0 2024-11-09T09:17:29,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741899_1075 (size=5049) 2024-11-09T09:17:29,260 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-09T09:17:29,260 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:29,461 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 1fbc66753606036aa3753c426d2e6336, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:29,557 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/.tmp/testFamily/b4137d7ae6114c87bb415ebd37d74e40 2024-11-09T09:17:29,573 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/.tmp/testFamily/b4137d7ae6114c87bb415ebd37d74e40 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/testFamily/b4137d7ae6114c87bb415ebd37d74e40 2024-11-09T09:17:29,581 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/testFamily/b4137d7ae6114c87bb415ebd37d74e40, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:29,582 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1fbc66753606036aa3753c426d2e6336 in 449ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:29,587 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnClose_1_-true/1fbc66753606036aa3753c426d2e6336/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:29,587 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:17:29,587 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1fbc66753606036aa3753c426d2e6336: 2024-11-09T09:17:29,587 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731143786521.1fbc66753606036aa3753c426d2e6336. 2024-11-09T09:17:29,588 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2ed876c86d7febba0ebb3221271b6b23, disabling compactions & flushes 2024-11-09T09:17:29,588 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:17:29,588 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:17:29,588 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. after waiting 0 ms 2024-11-09T09:17:29,588 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:17:29,588 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2ed876c86d7febba0ebb3221271b6b23 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:29,607 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/.tmp/testFamily/35232c27cb92469bb8e10321f1b0dffb is 61, key is row-0/testFamily:testQualifier/1731143681465/Put/seqid=0 2024-11-09T09:17:29,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741900_1076 (size=5049) 2024-11-09T09:17:29,661 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:29,862 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2ed876c86d7febba0ebb3221271b6b23, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:30,013 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/.tmp/testFamily/35232c27cb92469bb8e10321f1b0dffb 2024-11-09T09:17:30,029 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/.tmp/testFamily/35232c27cb92469bb8e10321f1b0dffb as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/testFamily/35232c27cb92469bb8e10321f1b0dffb 2024-11-09T09:17:30,038 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/testFamily/35232c27cb92469bb8e10321f1b0dffb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:30,039 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ed876c86d7febba0ebb3221271b6b23 in 451ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:30,043 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_0_-false/2ed876c86d7febba0ebb3221271b6b23/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:30,043 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:17:30,043 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2ed876c86d7febba0ebb3221271b6b23: 2024-11-09T09:17:30,044 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731143678436.2ed876c86d7febba0ebb3221271b6b23. 2024-11-09T09:17:30,044 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3a971eb4fc571ad608de182c691878da, disabling compactions & flushes 2024-11-09T09:17:30,044 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:30,044 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:30,044 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. after waiting 0 ms 2024-11-09T09:17:30,044 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:30,044 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3a971eb4fc571ad608de182c691878da 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:30,059 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/.tmp/testFamily/d837118447384d3cbf26c97f6a4cec35 is 61, key is row-0/testFamily:testQualifier/1731143769001/Put/seqid=0 2024-11-09T09:17:30,062 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:30,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741901_1077 (size=5049) 2024-11-09T09:17:30,262 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-09T09:17:30,262 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:30,463 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 3a971eb4fc571ad608de182c691878da, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:30,464 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/.tmp/testFamily/d837118447384d3cbf26c97f6a4cec35 2024-11-09T09:17:30,476 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/.tmp/testFamily/d837118447384d3cbf26c97f6a4cec35 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/testFamily/d837118447384d3cbf26c97f6a4cec35 2024-11-09T09:17:30,483 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/testFamily/d837118447384d3cbf26c97f6a4cec35, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:30,484 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3a971eb4fc571ad608de182c691878da in 440ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_1_-true/3a971eb4fc571ad608de182c691878da/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:30,488 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3a971eb4fc571ad608de182c691878da: 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731143765972.3a971eb4fc571ad608de182c691878da. 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0699f50861d7de54ae4352ede8ba637f, disabling compactions & flushes 2024-11-09T09:17:30,488 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. after waiting 0 ms 2024-11-09T09:17:30,488 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:30,488 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0699f50861d7de54ae4352ede8ba637f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:30,503 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/.tmp/testFamily/9924a37079a84274a13615117687cbca is 61, key is row-0/testFamily:testQualifier/1731143832108/Put/seqid=0 2024-11-09T09:17:30,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741902_1078 (size=5049) 2024-11-09T09:17:30,663 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:30,863 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 0699f50861d7de54ae4352ede8ba637f, 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:30,909 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/.tmp/testFamily/9924a37079a84274a13615117687cbca 2024-11-09T09:17:30,921 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/.tmp/testFamily/9924a37079a84274a13615117687cbca as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/testFamily/9924a37079a84274a13615117687cbca 2024-11-09T09:17:30,927 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/testFamily/9924a37079a84274a13615117687cbca, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:30,928 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0699f50861d7de54ae4352ede8ba637f in 440ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:30,932 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_1_-true/0699f50861d7de54ae4352ede8ba637f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:30,933 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:30,933 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0699f50861d7de54ae4352ede8ba637f: 2024-11-09T09:17:30,933 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731143829079.0699f50861d7de54ae4352ede8ba637f. 2024-11-09T09:17:30,933 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 271a1f8adda9f40e39312afa694ac217, disabling compactions & flushes 2024-11-09T09:17:30,933 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:17:30,933 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:17:30,933 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. after waiting 0 ms 2024-11-09T09:17:30,933 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:17:30,933 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 271a1f8adda9f40e39312afa694ac217 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:30,948 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/.tmp/testFamily/481f8751dad74a21b9983f572eacd3d4 is 61, key is row-0/testFamily:testQualifier/1731143706026/Put/seqid=0 2024-11-09T09:17:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741903_1079 (size=5049) 2024-11-09T09:17:31,064 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:31,264 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-09T09:17:31,264 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 271a1f8adda9f40e39312afa694ac217, 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:31,354 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/.tmp/testFamily/481f8751dad74a21b9983f572eacd3d4 2024-11-09T09:17:31,368 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/.tmp/testFamily/481f8751dad74a21b9983f572eacd3d4 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/testFamily/481f8751dad74a21b9983f572eacd3d4 2024-11-09T09:17:31,376 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/testFamily/481f8751dad74a21b9983f572eacd3d4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:31,377 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 271a1f8adda9f40e39312afa694ac217 in 444ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:31,381 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/271a1f8adda9f40e39312afa694ac217/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:31,382 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:17:31,382 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 271a1f8adda9f40e39312afa694ac217: 2024-11-09T09:17:31,382 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731143702994.271a1f8adda9f40e39312afa694ac217. 2024-11-09T09:17:31,382 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ca546e27319aa177954e402fd053bd2b, disabling compactions & flushes 2024-11-09T09:17:31,382 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:17:31,382 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:17:31,382 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. after waiting 0 ms 2024-11-09T09:17:31,382 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:17:31,382 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ca546e27319aa177954e402fd053bd2b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:31,397 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/.tmp/testFamily/9def13a470114d118c88c9faa795ba87 is 61, key is row-0/testFamily:testQualifier/1731143642640/Put/seqid=0 2024-11-09T09:17:31,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741904_1080 (size=5049) 2024-11-09T09:17:31,464 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:31,665 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, ca546e27319aa177954e402fd053bd2b, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:31,804 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/.tmp/testFamily/9def13a470114d118c88c9faa795ba87 2024-11-09T09:17:31,818 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/.tmp/testFamily/9def13a470114d118c88c9faa795ba87 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/testFamily/9def13a470114d118c88c9faa795ba87 2024-11-09T09:17:31,824 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/testFamily/9def13a470114d118c88c9faa795ba87, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:31,825 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ca546e27319aa177954e402fd053bd2b in 443ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:31,829 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ca546e27319aa177954e402fd053bd2b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:31,829 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:17:31,829 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ca546e27319aa177954e402fd053bd2b: 2024-11-09T09:17:31,829 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731143641611.ca546e27319aa177954e402fd053bd2b. 2024-11-09T09:17:31,829 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 316c4850c2b79d7cbe87367fe6cd94b3, disabling compactions & flushes 2024-11-09T09:17:31,829 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:17:31,829 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:17:31,830 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. after waiting 0 ms 2024-11-09T09:17:31,830 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:17:31,830 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 316c4850c2b79d7cbe87367fe6cd94b3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:31,844 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/.tmp/testFamily/f9ed0c3ed3d2402da9a7e2f43182c182 is 61, key is row-0/testFamily:testQualifier/1731143772098/Put/seqid=0 2024-11-09T09:17:31,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741905_1081 (size=5049) 2024-11-09T09:17:31,865 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:32,065 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:32,250 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/.tmp/testFamily/f9ed0c3ed3d2402da9a7e2f43182c182 2024-11-09T09:17:32,266 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 19 regions to close 2024-11-09T09:17:32,266 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 316c4850c2b79d7cbe87367fe6cd94b3, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:32,285 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/.tmp/testFamily/f9ed0c3ed3d2402da9a7e2f43182c182 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/testFamily/f9ed0c3ed3d2402da9a7e2f43182c182 2024-11-09T09:17:32,293 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/testFamily/f9ed0c3ed3d2402da9a7e2f43182c182, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:32,294 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 316c4850c2b79d7cbe87367fe6cd94b3 in 464ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:32,298 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_1_-true/316c4850c2b79d7cbe87367fe6cd94b3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:32,299 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:17:32,299 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 316c4850c2b79d7cbe87367fe6cd94b3: 2024-11-09T09:17:32,299 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731143769071.316c4850c2b79d7cbe87367fe6cd94b3. 2024-11-09T09:17:32,299 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 496f9744c3b0aceac24ffc23bd44774f, disabling compactions & flushes 2024-11-09T09:17:32,299 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:17:32,299 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:17:32,299 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. after waiting 0 ms 2024-11-09T09:17:32,299 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:17:32,299 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 496f9744c3b0aceac24ffc23bd44774f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:32,314 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/.tmp/testFamily/f1e0eddf1ef14c7d89b2c8a6900688cb is 61, key is row-0/testFamily:testQualifier/1731143675292/Put/seqid=0 2024-11-09T09:17:32,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741906_1082 (size=5049) 2024-11-09T09:17:32,466 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:32,667 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 496f9744c3b0aceac24ffc23bd44774f, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:32,720 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/.tmp/testFamily/f1e0eddf1ef14c7d89b2c8a6900688cb 2024-11-09T09:17:32,734 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/.tmp/testFamily/f1e0eddf1ef14c7d89b2c8a6900688cb as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/testFamily/f1e0eddf1ef14c7d89b2c8a6900688cb 2024-11-09T09:17:32,744 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/testFamily/f1e0eddf1ef14c7d89b2c8a6900688cb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:32,745 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 496f9744c3b0aceac24ffc23bd44774f in 446ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:32,750 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/496f9744c3b0aceac24ffc23bd44774f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:32,750 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:17:32,750 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 496f9744c3b0aceac24ffc23bd44774f: 2024-11-09T09:17:32,751 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731143672260.496f9744c3b0aceac24ffc23bd44774f. 2024-11-09T09:17:32,751 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 999c6baaaa000b15d3dbb1d6dc21b2c4, disabling compactions & flushes 2024-11-09T09:17:32,751 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:17:32,751 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:17:32,751 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. after waiting 0 ms 2024-11-09T09:17:32,751 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:17:32,751 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 999c6baaaa000b15d3dbb1d6dc21b2c4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:32,769 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/.tmp/testFamily/310976bd86bf4ecfac9d97631a12c0c0 is 61, key is row-0/testFamily:testQualifier/1731143809853/Put/seqid=0 2024-11-09T09:17:32,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741907_1083 (size=5049) 2024-11-09T09:17:32,867 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:33,067 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, 999c6baaaa000b15d3dbb1d6dc21b2c4, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:33,175 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/.tmp/testFamily/310976bd86bf4ecfac9d97631a12c0c0 2024-11-09T09:17:33,189 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/.tmp/testFamily/310976bd86bf4ecfac9d97631a12c0c0 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/testFamily/310976bd86bf4ecfac9d97631a12c0c0 2024-11-09T09:17:33,195 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/testFamily/310976bd86bf4ecfac9d97631a12c0c0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:33,196 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 999c6baaaa000b15d3dbb1d6dc21b2c4 in 445ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:33,200 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/999c6baaaa000b15d3dbb1d6dc21b2c4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:33,201 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:17:33,201 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 999c6baaaa000b15d3dbb1d6dc21b2c4: 2024-11-09T09:17:33,201 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731143806835.999c6baaaa000b15d3dbb1d6dc21b2c4. 2024-11-09T09:17:33,201 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9b80974f1b4163166250b30045e5a88b, disabling compactions & flushes 2024-11-09T09:17:33,201 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:17:33,201 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:17:33,201 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. after waiting 0 ms 2024-11-09T09:17:33,201 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:17:33,201 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9b80974f1b4163166250b30045e5a88b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:33,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/.tmp/testFamily/caabf2fb8ef64f8b8cc3ec0eeb6bfcad is 61, key is row-0/testFamily:testQualifier/1731143728320/Put/seqid=0 2024-11-09T09:17:33,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741908_1084 (size=5049) 2024-11-09T09:17:33,268 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-09T09:17:33,268 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:33,285 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T09:17:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-09T09:17:33,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-09T09:17:33,468 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, 9b80974f1b4163166250b30045e5a88b, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:33,624 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/.tmp/testFamily/caabf2fb8ef64f8b8cc3ec0eeb6bfcad 2024-11-09T09:17:33,636 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/.tmp/testFamily/caabf2fb8ef64f8b8cc3ec0eeb6bfcad as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/testFamily/caabf2fb8ef64f8b8cc3ec0eeb6bfcad 2024-11-09T09:17:33,643 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/testFamily/caabf2fb8ef64f8b8cc3ec0eeb6bfcad, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:33,644 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b80974f1b4163166250b30045e5a88b in 443ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:33,648 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNext_0_-false/9b80974f1b4163166250b30045e5a88b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:33,649 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:17:33,649 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9b80974f1b4163166250b30045e5a88b: 2024-11-09T09:17:33,649 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731143725289.9b80974f1b4163166250b30045e5a88b. 2024-11-09T09:17:33,649 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d8faed9fa602af320f8ab06bb017e46d, disabling compactions & flushes 2024-11-09T09:17:33,649 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:17:33,649 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:17:33,649 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. after waiting 0 ms 2024-11-09T09:17:33,649 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:17:33,649 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d8faed9fa602af320f8ab06bb017e46d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:33,665 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/.tmp/testFamily/5eb2b6c5fdfb484d833d2367833ac4a0 is 61, key is row-0/testFamily:testQualifier/1731143783410/Put/seqid=0 2024-11-09T09:17:33,668 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:33,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741909_1085 (size=5049) 2024-11-09T09:17:33,869 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:34,069 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, d8faed9fa602af320f8ab06bb017e46d, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:34,070 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/.tmp/testFamily/5eb2b6c5fdfb484d833d2367833ac4a0 2024-11-09T09:17:34,081 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/.tmp/testFamily/5eb2b6c5fdfb484d833d2367833ac4a0 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/testFamily/5eb2b6c5fdfb484d833d2367833ac4a0 2024-11-09T09:17:34,087 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/testFamily/5eb2b6c5fdfb484d833d2367833ac4a0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:34,087 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d8faed9fa602af320f8ab06bb017e46d in 438ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:34,091 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNextAsync_1_-true/d8faed9fa602af320f8ab06bb017e46d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:34,092 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:17:34,092 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d8faed9fa602af320f8ab06bb017e46d: 2024-11-09T09:17:34,092 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731143780380.d8faed9fa602af320f8ab06bb017e46d. 2024-11-09T09:17:34,092 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 36ec248a87a5f6a35a3be224ec1de4c5, disabling compactions & flushes 2024-11-09T09:17:34,092 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:17:34,092 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:17:34,092 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. after waiting 0 ms 2024-11-09T09:17:34,092 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:17:34,092 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 36ec248a87a5f6a35a3be224ec1de4c5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:34,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/.tmp/testFamily/58f3ca0994324f788249ff7586df3817 is 61, key is row-0/testFamily:testQualifier/1731143737488/Put/seqid=0 2024-11-09T09:17:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741910_1086 (size=5049) 2024-11-09T09:17:34,269 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 14 regions to close 2024-11-09T09:17:34,269 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:34,470 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 36ec248a87a5f6a35a3be224ec1de4c5, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:34,513 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/.tmp/testFamily/58f3ca0994324f788249ff7586df3817 2024-11-09T09:17:34,526 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/.tmp/testFamily/58f3ca0994324f788249ff7586df3817 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/testFamily/58f3ca0994324f788249ff7586df3817 2024-11-09T09:17:34,533 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/testFamily/58f3ca0994324f788249ff7586df3817, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:34,534 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 36ec248a87a5f6a35a3be224ec1de4c5 in 442ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:34,538 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/36ec248a87a5f6a35a3be224ec1de4c5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:34,539 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:17:34,539 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 36ec248a87a5f6a35a3be224ec1de4c5: 2024-11-09T09:17:34,539 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731143734460.36ec248a87a5f6a35a3be224ec1de4c5. 2024-11-09T09:17:34,539 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c4fd6ba823547d504f5d98c1c026399e, disabling compactions & flushes 2024-11-09T09:17:34,539 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:17:34,539 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:17:34,539 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. after waiting 0 ms 2024-11-09T09:17:34,539 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:17:34,539 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c4fd6ba823547d504f5d98c1c026399e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:34,556 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/.tmp/testFamily/ad2c33015c1d4438a73325cded639d8d is 61, key is row-0/testFamily:testQualifier/1731143716170/Put/seqid=0 2024-11-09T09:17:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741911_1087 (size=5049) 2024-11-09T09:17:34,670 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:34,871 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, c4fd6ba823547d504f5d98c1c026399e, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:34,961 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/.tmp/testFamily/ad2c33015c1d4438a73325cded639d8d 2024-11-09T09:17:34,973 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/.tmp/testFamily/ad2c33015c1d4438a73325cded639d8d as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/testFamily/ad2c33015c1d4438a73325cded639d8d 2024-11-09T09:17:34,981 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/testFamily/ad2c33015c1d4438a73325cded639d8d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:34,982 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c4fd6ba823547d504f5d98c1c026399e in 443ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:34,986 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c4fd6ba823547d504f5d98c1c026399e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:34,986 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:17:34,986 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c4fd6ba823547d504f5d98c1c026399e: 2024-11-09T09:17:34,986 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731143713137.c4fd6ba823547d504f5d98c1c026399e. 2024-11-09T09:17:34,987 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9052953756601a7a9f875737f91f1fa6, disabling compactions & flushes 2024-11-09T09:17:34,987 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:34,987 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:34,987 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. after waiting 0 ms 2024-11-09T09:17:34,987 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:34,987 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9052953756601a7a9f875737f91f1fa6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:35,000 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/.tmp/testFamily/1389db2837784c43a114b19bd10b8ac3 is 61, key is row-0/testFamily:testQualifier/1731143844206/Put/seqid=0 2024-11-09T09:17:35,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741912_1088 (size=5049) 2024-11-09T09:17:35,071 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:35,271 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 12 regions to close 2024-11-09T09:17:35,271 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, 9052953756601a7a9f875737f91f1fa6, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:35,406 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/.tmp/testFamily/1389db2837784c43a114b19bd10b8ac3 2024-11-09T09:17:35,420 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/.tmp/testFamily/1389db2837784c43a114b19bd10b8ac3 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/testFamily/1389db2837784c43a114b19bd10b8ac3 2024-11-09T09:17:35,428 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/testFamily/1389db2837784c43a114b19bd10b8ac3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:35,429 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9052953756601a7a9f875737f91f1fa6 in 442ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:35,433 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/9052953756601a7a9f875737f91f1fa6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:35,434 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9052953756601a7a9f875737f91f1fa6: 2024-11-09T09:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731143841178.9052953756601a7a9f875737f91f1fa6. 2024-11-09T09:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ef34766cd3a29a746f1b7db5199d51e4, disabling compactions & flushes 2024-11-09T09:17:35,434 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. after waiting 0 ms 2024-11-09T09:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:17:35,434 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ef34766cd3a29a746f1b7db5199d51e4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:35,449 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/.tmp/testFamily/975e5ed6bddc449b8c8e09f2b769fd13 is 61, key is row-0/testFamily:testQualifier/1731143741638/Put/seqid=0 2024-11-09T09:17:35,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741913_1089 (size=5049) 2024-11-09T09:17:35,472 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:35,672 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:35,855 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/.tmp/testFamily/975e5ed6bddc449b8c8e09f2b769fd13 2024-11-09T09:17:35,867 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/.tmp/testFamily/975e5ed6bddc449b8c8e09f2b769fd13 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/testFamily/975e5ed6bddc449b8c8e09f2b769fd13 2024-11-09T09:17:35,872 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, ef34766cd3a29a746f1b7db5199d51e4, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:35,874 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/testFamily/975e5ed6bddc449b8c8e09f2b769fd13, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:35,875 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ef34766cd3a29a746f1b7db5199d51e4 in 441ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:35,879 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/ef34766cd3a29a746f1b7db5199d51e4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:35,880 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:17:35,880 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ef34766cd3a29a746f1b7db5199d51e4: 2024-11-09T09:17:35,880 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731143740610.ef34766cd3a29a746f1b7db5199d51e4. 2024-11-09T09:17:35,880 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f6ef5e815e8a0538da916d41f9c6bba5, disabling compactions & flushes 2024-11-09T09:17:35,880 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:17:35,880 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:17:35,880 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. after waiting 0 ms 2024-11-09T09:17:35,880 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:17:35,880 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f6ef5e815e8a0538da916d41f9c6bba5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:35,896 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/.tmp/testFamily/b7fe9cd32c09452bbbc6d2cada4c6ade is 61, key is row-0/testFamily:testQualifier/1731143666986/Put/seqid=0 2024-11-09T09:17:35,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741914_1090 (size=5049) 2024-11-09T09:17:36,072 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:36,273 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 10 regions to close 2024-11-09T09:17:36,273 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, f6ef5e815e8a0538da916d41f9c6bba5, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:36,301 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/.tmp/testFamily/b7fe9cd32c09452bbbc6d2cada4c6ade 2024-11-09T09:17:36,307 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/.tmp/testFamily/b7fe9cd32c09452bbbc6d2cada4c6ade as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/testFamily/b7fe9cd32c09452bbbc6d2cada4c6ade 2024-11-09T09:17:36,313 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/testFamily/b7fe9cd32c09452bbbc6d2cada4c6ade, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:36,314 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f6ef5e815e8a0538da916d41f9c6bba5 in 434ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:36,319 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextException_0_-false/f6ef5e815e8a0538da916d41f9c6bba5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:36,320 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:17:36,320 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f6ef5e815e8a0538da916d41f9c6bba5: 2024-11-09T09:17:36,320 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731143663954.f6ef5e815e8a0538da916d41f9c6bba5. 2024-11-09T09:17:36,320 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d29da8fc7e2c0292000b4931ecdf1ed1, disabling compactions & flushes 2024-11-09T09:17:36,320 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:17:36,320 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:17:36,320 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. after waiting 0 ms 2024-11-09T09:17:36,320 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:17:36,320 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d29da8fc7e2c0292000b4931ecdf1ed1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:36,339 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/.tmp/testFamily/953238d2884a49009db3f8c4ddc0d4d4 is 61, key is row-0/testFamily:testQualifier/1731143744741/Put/seqid=0 2024-11-09T09:17:36,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741915_1091 (size=5049) 2024-11-09T09:17:36,473 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:36,674 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, d29da8fc7e2c0292000b4931ecdf1ed1, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:36,745 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/.tmp/testFamily/953238d2884a49009db3f8c4ddc0d4d4 2024-11-09T09:17:36,757 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/.tmp/testFamily/953238d2884a49009db3f8c4ddc0d4d4 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/testFamily/953238d2884a49009db3f8c4ddc0d4d4 2024-11-09T09:17:36,764 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/testFamily/953238d2884a49009db3f8c4ddc0d4d4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:36,765 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d29da8fc7e2c0292000b4931ecdf1ed1 in 445ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:36,770 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/d29da8fc7e2c0292000b4931ecdf1ed1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:36,771 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:17:36,771 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d29da8fc7e2c0292000b4931ecdf1ed1: 2024-11-09T09:17:36,771 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731143741710.d29da8fc7e2c0292000b4931ecdf1ed1. 2024-11-09T09:17:36,771 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ba96f0895981853116801be5af09f3d2, disabling compactions & flushes 2024-11-09T09:17:36,771 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:17:36,771 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:17:36,771 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. after waiting 0 ms 2024-11-09T09:17:36,771 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:17:36,771 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ba96f0895981853116801be5af09f3d2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:36,789 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/.tmp/testFamily/285874bca599456585f6fe341c6cbe6d is 61, key is row-0/testFamily:testQualifier/1731143697217/Put/seqid=0 2024-11-09T09:17:36,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741916_1092 (size=5049) 2024-11-09T09:17:36,874 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:37,075 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, ba96f0895981853116801be5af09f3d2, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:37,194 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/.tmp/testFamily/285874bca599456585f6fe341c6cbe6d 2024-11-09T09:17:37,204 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/.tmp/testFamily/285874bca599456585f6fe341c6cbe6d as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/testFamily/285874bca599456585f6fe341c6cbe6d 2024-11-09T09:17:37,213 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/testFamily/285874bca599456585f6fe341c6cbe6d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:37,214 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ba96f0895981853116801be5af09f3d2 in 443ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:37,217 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnClose_0_-false/ba96f0895981853116801be5af09f3d2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:37,218 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:17:37,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ba96f0895981853116801be5af09f3d2: 2024-11-09T09:17:37,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731143694185.ba96f0895981853116801be5af09f3d2. 2024-11-09T09:17:37,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 459f96eb1d922bcd9f696d43a6691e8c, disabling compactions & flushes 2024-11-09T09:17:37,218 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:17:37,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:17:37,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. after waiting 0 ms 2024-11-09T09:17:37,218 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:17:37,218 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 459f96eb1d922bcd9f696d43a6691e8c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:37,233 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/.tmp/testFamily/dde95d25db774cbebfb2417ce076e40d is 61, key is row-0/testFamily:testQualifier/1731143641459/Put/seqid=0 2024-11-09T09:17:37,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741917_1093 (size=5049) 2024-11-09T09:17:37,275 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 7 regions to close 2024-11-09T09:17:37,275 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1603): Online Regions={459f96eb1d922bcd9f696d43a6691e8c=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c., cc68045eb729be737b2bb40a8dcab3f4=hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4., f90ee672bf70ce14e71d8eba9060419a=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a., b502209f5d68712d0c9e5af7a60429f6=testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6., abf768636ae7ebb0ba6696eb118fa2ba=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., 46c966491675fb74bc449a58cad47886=testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886., 2fb42412d02f8c07a7a7398fa2e5222e=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.} 2024-11-09T09:17:37,275 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:37,475 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 459f96eb1d922bcd9f696d43a6691e8c, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:37,639 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/.tmp/testFamily/dde95d25db774cbebfb2417ce076e40d 2024-11-09T09:17:37,652 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/.tmp/testFamily/dde95d25db774cbebfb2417ce076e40d as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/testFamily/dde95d25db774cbebfb2417ce076e40d 2024-11-09T09:17:37,658 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/testFamily/dde95d25db774cbebfb2417ce076e40d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:37,659 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 459f96eb1d922bcd9f696d43a6691e8c in 441ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:37,662 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/459f96eb1d922bcd9f696d43a6691e8c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:37,663 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:17:37,663 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 459f96eb1d922bcd9f696d43a6691e8c: 2024-11-09T09:17:37,663 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731143638408.459f96eb1d922bcd9f696d43a6691e8c. 2024-11-09T09:17:37,663 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing cc68045eb729be737b2bb40a8dcab3f4, disabling compactions & flushes 2024-11-09T09:17:37,663 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:17:37,663 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:17:37,663 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. after waiting 0 ms 2024-11-09T09:17:37,663 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:17:37,663 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing cc68045eb729be737b2bb40a8dcab3f4 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-09T09:17:37,676 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:37,679 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/.tmp/info/b54adac5881a44669abd343fb417f430 is 45, key is default/info:d/1731143638150/Put/seqid=0 2024-11-09T09:17:37,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741918_1094 (size=5037) 2024-11-09T09:17:37,876 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:38,076 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, cc68045eb729be737b2bb40a8dcab3f4, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:38,085 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/.tmp/info/b54adac5881a44669abd343fb417f430 2024-11-09T09:17:38,095 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/.tmp/info/b54adac5881a44669abd343fb417f430 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/info/b54adac5881a44669abd343fb417f430 2024-11-09T09:17:38,102 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/info/b54adac5881a44669abd343fb417f430, entries=2, sequenceid=6, filesize=4.9 K 2024-11-09T09:17:38,102 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for cc68045eb729be737b2bb40a8dcab3f4 in 439ms, sequenceid=6, compaction requested=false 2024-11-09T09:17:38,106 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/namespace/cc68045eb729be737b2bb40a8dcab3f4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-09T09:17:38,107 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:17:38,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for cc68045eb729be737b2bb40a8dcab3f4: 2024-11-09T09:17:38,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731143636686.cc68045eb729be737b2bb40a8dcab3f4. 2024-11-09T09:17:38,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f90ee672bf70ce14e71d8eba9060419a, disabling compactions & flushes 2024-11-09T09:17:38,107 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:17:38,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:17:38,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. after waiting 0 ms 2024-11-09T09:17:38,107 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:17:38,107 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f90ee672bf70ce14e71d8eba9060419a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:38,122 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/.tmp/testFamily/bf60fc9d52924cbc8396e3326434a90e is 61, key is row-0/testFamily:testQualifier/1731143654813/Put/seqid=0 2024-11-09T09:17:38,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741919_1095 (size=5049) 2024-11-09T09:17:38,253 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T09:17:38,276 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 5 regions to close 2024-11-09T09:17:38,277 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1603): Online Regions={f90ee672bf70ce14e71d8eba9060419a=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a., b502209f5d68712d0c9e5af7a60429f6=testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6., abf768636ae7ebb0ba6696eb118fa2ba=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., 46c966491675fb74bc449a58cad47886=testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886., 2fb42412d02f8c07a7a7398fa2e5222e=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.} 2024-11-09T09:17:38,277 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:38,477 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6, f90ee672bf70ce14e71d8eba9060419a 2024-11-09T09:17:38,527 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/.tmp/testFamily/bf60fc9d52924cbc8396e3326434a90e 2024-11-09T09:17:38,542 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/.tmp/testFamily/bf60fc9d52924cbc8396e3326434a90e as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/testFamily/bf60fc9d52924cbc8396e3326434a90e 2024-11-09T09:17:38,548 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/testFamily/bf60fc9d52924cbc8396e3326434a90e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:38,549 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f90ee672bf70ce14e71d8eba9060419a in 441ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:38,552 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f90ee672bf70ce14e71d8eba9060419a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:38,553 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:17:38,553 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f90ee672bf70ce14e71d8eba9060419a: 2024-11-09T09:17:38,553 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731143651777.f90ee672bf70ce14e71d8eba9060419a. 2024-11-09T09:17:38,553 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b502209f5d68712d0c9e5af7a60429f6, disabling compactions & flushes 2024-11-09T09:17:38,553 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:17:38,553 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:17:38,553 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. after waiting 0 ms 2024-11-09T09:17:38,553 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:17:38,553 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b502209f5d68712d0c9e5af7a60429f6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:38,570 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/.tmp/testFamily/d853cf2ca49149e28000885d3e9eb94f is 61, key is row-0/testFamily:testQualifier/1731143701854/Put/seqid=0 2024-11-09T09:17:38,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741920_1096 (size=5049) 2024-11-09T09:17:38,677 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:17:38,878 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba, b502209f5d68712d0c9e5af7a60429f6 2024-11-09T09:17:38,976 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/.tmp/testFamily/d853cf2ca49149e28000885d3e9eb94f 2024-11-09T09:17:38,989 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/.tmp/testFamily/d853cf2ca49149e28000885d3e9eb94f as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/testFamily/d853cf2ca49149e28000885d3e9eb94f 2024-11-09T09:17:38,997 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/testFamily/d853cf2ca49149e28000885d3e9eb94f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:38,998 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b502209f5d68712d0c9e5af7a60429f6 in 445ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:39,002 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnNext_0_-false/b502209f5d68712d0c9e5af7a60429f6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:39,003 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:17:39,003 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b502209f5d68712d0c9e5af7a60429f6: 2024-11-09T09:17:39,003 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731143698822.b502209f5d68712d0c9e5af7a60429f6. 2024-11-09T09:17:39,003 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing abf768636ae7ebb0ba6696eb118fa2ba, disabling compactions & flushes 2024-11-09T09:17:39,003 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:17:39,003 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:17:39,003 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. after waiting 0 ms 2024-11-09T09:17:39,003 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:17:39,003 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing abf768636ae7ebb0ba6696eb118fa2ba 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:39,017 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/.tmp/testFamily/a2d63a9de222412ab0ca1f4586607331 is 61, key is row-0/testFamily:testQualifier/1731143819977/Put/seqid=0 2024-11-09T09:17:39,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741921_1097 (size=5049) 2024-11-09T09:17:39,078 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:17:39,089 ERROR [LeaseRenewer:jenkins@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39925,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:17:39,278 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-11-09T09:17:39,278 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1603): Online Regions={abf768636ae7ebb0ba6696eb118fa2ba=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba., 46c966491675fb74bc449a58cad47886=testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886., 2fb42412d02f8c07a7a7398fa2e5222e=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.} 2024-11-09T09:17:39,278 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886, abf768636ae7ebb0ba6696eb118fa2ba 2024-11-09T09:17:39,423 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/.tmp/testFamily/a2d63a9de222412ab0ca1f4586607331 2024-11-09T09:17:39,434 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/.tmp/testFamily/a2d63a9de222412ab0ca1f4586607331 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/testFamily/a2d63a9de222412ab0ca1f4586607331 2024-11-09T09:17:39,441 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/testFamily/a2d63a9de222412ab0ca1f4586607331, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:39,442 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for abf768636ae7ebb0ba6696eb118fa2ba in 439ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:39,446 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/abf768636ae7ebb0ba6696eb118fa2ba/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:39,447 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:17:39,447 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for abf768636ae7ebb0ba6696eb118fa2ba: 2024-11-09T09:17:39,447 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731143816952.abf768636ae7ebb0ba6696eb118fa2ba. 2024-11-09T09:17:39,447 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 46c966491675fb74bc449a58cad47886, disabling compactions & flushes 2024-11-09T09:17:39,447 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:17:39,447 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:17:39,447 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. after waiting 0 ms 2024-11-09T09:17:39,447 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:17:39,447 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 46c966491675fb74bc449a58cad47886 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:39,464 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/.tmp/testFamily/62f42d30cdd44aa093ff7f4bf81cdb05 is 61, key is row-0/testFamily:testQualifier/1731143670116/Put/seqid=0 2024-11-09T09:17:39,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741922_1098 (size=5049) 2024-11-09T09:17:39,479 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886 2024-11-09T09:17:39,679 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886 2024-11-09T09:17:39,871 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/.tmp/testFamily/62f42d30cdd44aa093ff7f4bf81cdb05 2024-11-09T09:17:39,879 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e, 46c966491675fb74bc449a58cad47886 2024-11-09T09:17:39,886 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/.tmp/testFamily/62f42d30cdd44aa093ff7f4bf81cdb05 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/testFamily/62f42d30cdd44aa093ff7f4bf81cdb05 2024-11-09T09:17:39,895 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/testFamily/62f42d30cdd44aa093ff7f4bf81cdb05, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:39,896 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46c966491675fb74bc449a58cad47886 in 449ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:39,901 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/default/testNormalScanTimeoutOnRenewLease_0_-false/46c966491675fb74bc449a58cad47886/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:39,902 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:17:39,902 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 46c966491675fb74bc449a58cad47886: 2024-11-09T09:17:39,902 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731143667085.46c966491675fb74bc449a58cad47886. 2024-11-09T09:17:39,902 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2fb42412d02f8c07a7a7398fa2e5222e, disabling compactions & flushes 2024-11-09T09:17:39,902 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:17:39,902 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:17:39,902 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. after waiting 0 ms 2024-11-09T09:17:39,902 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:17:39,902 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2fb42412d02f8c07a7a7398fa2e5222e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T09:17:39,917 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/.tmp/testFamily/b4518f6c93674b3e88f1000683910684 is 61, key is row-0/testFamily:testQualifier/1731143756852/Put/seqid=0 2024-11-09T09:17:39,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741923_1099 (size=5049) 2024-11-09T09:17:40,080 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:17:40,280 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1599): Waiting on 1 regions to close 2024-11-09T09:17:40,280 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1603): Online Regions={2fb42412d02f8c07a7a7398fa2e5222e=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e.} 2024-11-09T09:17:40,280 DEBUG [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1629): Waiting on 2fb42412d02f8c07a7a7398fa2e5222e 2024-11-09T09:17:40,304 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39925 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39925,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T09:17:40,324 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/.tmp/testFamily/b4518f6c93674b3e88f1000683910684 2024-11-09T09:17:40,337 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/.tmp/testFamily/b4518f6c93674b3e88f1000683910684 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/testFamily/b4518f6c93674b3e88f1000683910684 2024-11-09T09:17:40,346 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/testFamily/b4518f6c93674b3e88f1000683910684, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T09:17:40,347 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2fb42412d02f8c07a7a7398fa2e5222e in 445ms, sequenceid=8, compaction requested=false 2024-11-09T09:17:40,351 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/2fb42412d02f8c07a7a7398fa2e5222e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T09:17:40,352 INFO [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:17:40,352 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2fb42412d02f8c07a7a7398fa2e5222e: 2024-11-09T09:17:40,352 DEBUG [RS_CLOSE_REGION-regionserver/9b029db41fd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731143753821.2fb42412d02f8c07a7a7398fa2e5222e. 2024-11-09T09:17:40,480 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1250): stopping server 9b029db41fd9,43449,1731143603787; all regions closed. 2024-11-09T09:17:40,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741834_1010 (size=56731) 2024-11-09T09:17:40,489 DEBUG [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/oldWALs 2024-11-09T09:17:40,489 INFO [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 9b029db41fd9%2C43449%2C1731143603787.meta:.meta(num 1731143636404) 2024-11-09T09:17:40,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741833_1009 (size=60562) 2024-11-09T09:17:40,494 DEBUG [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/oldWALs 2024-11-09T09:17:40,494 INFO [RS:0;9b029db41fd9:43449 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 9b029db41fd9%2C43449%2C1731143603787:(num 1731143635765) 2024-11-09T09:17:40,494 DEBUG [RS:0;9b029db41fd9:43449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:40,494 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T09:17:40,494 INFO [RS:0;9b029db41fd9:43449 {}] hbase.ChoreService(370): Chore service for: regionserver/9b029db41fd9:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T09:17:40,494 INFO [regionserver/9b029db41fd9:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-09T09:17:40,495 INFO [RS:0;9b029db41fd9:43449 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43449 2024-11-09T09:17:40,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9b029db41fd9,43449,1731143603787 2024-11-09T09:17:40,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T09:17:40,541 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9b029db41fd9,43449,1731143603787] 2024-11-09T09:17:40,541 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 9b029db41fd9,43449,1731143603787; numProcessing=1 2024-11-09T09:17:40,549 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/9b029db41fd9,43449,1731143603787 already deleted, retry=false 2024-11-09T09:17:40,549 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 9b029db41fd9,43449,1731143603787 expired; onlineServers=0 2024-11-09T09:17:40,549 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '9b029db41fd9,41847,1731143603026' ***** 2024-11-09T09:17:40,549 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T09:17:40,550 DEBUG [M:0;9b029db41fd9:41847 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7329a93a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9b029db41fd9/172.17.0.2:0 2024-11-09T09:17:40,550 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegionServer(1224): stopping server 9b029db41fd9,41847,1731143603026 2024-11-09T09:17:40,550 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegionServer(1250): stopping server 9b029db41fd9,41847,1731143603026; all regions closed. 2024-11-09T09:17:40,550 DEBUG [M:0;9b029db41fd9:41847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T09:17:40,550 DEBUG [M:0;9b029db41fd9:41847 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T09:17:40,551 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T09:17:40,551 DEBUG [M:0;9b029db41fd9:41847 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T09:17:40,551 DEBUG [master/9b029db41fd9:0:becomeActiveMaster-HFileCleaner.large.0-1731143605577 {}] cleaner.HFileCleaner(306): Exit Thread[master/9b029db41fd9:0:becomeActiveMaster-HFileCleaner.large.0-1731143605577,5,FailOnTimeoutGroup] 2024-11-09T09:17:40,551 DEBUG [master/9b029db41fd9:0:becomeActiveMaster-HFileCleaner.small.0-1731143605577 {}] cleaner.HFileCleaner(306): Exit Thread[master/9b029db41fd9:0:becomeActiveMaster-HFileCleaner.small.0-1731143605577,5,FailOnTimeoutGroup] 2024-11-09T09:17:40,552 INFO [M:0;9b029db41fd9:41847 {}] hbase.ChoreService(370): Chore service for: master/9b029db41fd9:0 had [] on shutdown 2024-11-09T09:17:40,552 DEBUG [M:0;9b029db41fd9:41847 {}] master.HMaster(1733): Stopping service threads 2024-11-09T09:17:40,552 INFO [M:0;9b029db41fd9:41847 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T09:17:40,553 ERROR [M:0;9b029db41fd9:41847 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39925,5,PEWorkerGroup] 2024-11-09T09:17:40,555 INFO [M:0;9b029db41fd9:41847 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T09:17:40,555 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T09:17:40,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T09:17:40,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T09:17:40,558 DEBUG [M:0;9b029db41fd9:41847 {}] zookeeper.ZKUtil(347): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T09:17:40,558 WARN [M:0;9b029db41fd9:41847 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T09:17:40,558 INFO [M:0;9b029db41fd9:41847 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-09T09:17:40,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T09:17:40,559 INFO [M:0;9b029db41fd9:41847 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T09:17:40,559 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T09:17:40,559 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:17:40,559 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:17:40,560 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T09:17:40,560 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:17:40,560 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.63 KB heapSize=520.41 KB 2024-11-09T09:17:40,563 INFO [regionserver/9b029db41fd9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T09:17:40,578 DEBUG [M:0;9b029db41fd9:41847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/50bcf8ad741246cca909985c602d9253 is 82, key is hbase:meta,,1/info:regioninfo/1731143636519/Put/seqid=0 2024-11-09T09:17:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741924_1100 (size=5672) 2024-11-09T09:17:40,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T09:17:40,641 INFO [RS:0;9b029db41fd9:43449 {}] regionserver.HRegionServer(1307): Exiting; stopping=9b029db41fd9,43449,1731143603787; zookeeper connection closed. 2024-11-09T09:17:40,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43449-0x1012028b2df0001, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T09:17:40,641 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fc2c62e {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fc2c62e 2024-11-09T09:17:40,642 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T09:17:40,984 INFO [M:0;9b029db41fd9:41847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/50bcf8ad741246cca909985c602d9253 2024-11-09T09:17:41,015 DEBUG [M:0;9b029db41fd9:41847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d75daba685954ee0b1dc3fbdb2f7e537 is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731143639694/Put/seqid=0 2024-11-09T09:17:41,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741925_1101 (size=29527) 2024-11-09T09:17:41,422 INFO [M:0;9b029db41fd9:41847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=448.07 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d75daba685954ee0b1dc3fbdb2f7e537 2024-11-09T09:17:41,426 INFO [M:0;9b029db41fd9:41847 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d75daba685954ee0b1dc3fbdb2f7e537 2024-11-09T09:17:41,443 DEBUG [M:0;9b029db41fd9:41847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89f47cf88ede4fabbfe820061fb7aa82 is 69, key is 9b029db41fd9,43449,1731143603787/rs:state/1731143605617/Put/seqid=0 2024-11-09T09:17:41,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741926_1102 (size=5156) 2024-11-09T09:17:41,850 INFO [M:0;9b029db41fd9:41847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89f47cf88ede4fabbfe820061fb7aa82 2024-11-09T09:17:41,859 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/50bcf8ad741246cca909985c602d9253 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/50bcf8ad741246cca909985c602d9253 2024-11-09T09:17:41,864 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/50bcf8ad741246cca909985c602d9253, entries=8, sequenceid=834, filesize=5.5 K 2024-11-09T09:17:41,865 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d75daba685954ee0b1dc3fbdb2f7e537 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d75daba685954ee0b1dc3fbdb2f7e537 2024-11-09T09:17:41,870 INFO [M:0;9b029db41fd9:41847 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d75daba685954ee0b1dc3fbdb2f7e537 2024-11-09T09:17:41,870 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d75daba685954ee0b1dc3fbdb2f7e537, entries=92, sequenceid=834, filesize=28.8 K 2024-11-09T09:17:41,871 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89f47cf88ede4fabbfe820061fb7aa82 as hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89f47cf88ede4fabbfe820061fb7aa82 2024-11-09T09:17:41,875 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39925/user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89f47cf88ede4fabbfe820061fb7aa82, entries=1, sequenceid=834, filesize=5.0 K 2024-11-09T09:17:41,876 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.63 KB/459393, heapSize ~520.11 KB/532592, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=834, compaction requested=false 2024-11-09T09:17:41,877 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T09:17:41,877 DEBUG [M:0;9b029db41fd9:41847 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T09:17:41,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34159 is added to blk_1073741830_1006 (size=514522) 2024-11-09T09:17:41,879 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/b1a7e67f-f9b6-dd84-987a-e67a0320e014/MasterData/WALs/9b029db41fd9,41847,1731143603026/9b029db41fd9%2C41847%2C1731143603026.1731143604973 not finished, retry = 0 2024-11-09T09:17:41,981 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-09T09:17:41,981 INFO [M:0;9b029db41fd9:41847 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-09T09:17:41,982 INFO [M:0;9b029db41fd9:41847 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41847 2024-11-09T09:17:42,023 DEBUG [M:0;9b029db41fd9:41847 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/9b029db41fd9,41847,1731143603026 already deleted, retry=false 2024-11-09T09:17:42,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T09:17:42,133 INFO [M:0;9b029db41fd9:41847 {}] regionserver.HRegionServer(1307): Exiting; stopping=9b029db41fd9,41847,1731143603026; zookeeper connection closed. 2024-11-09T09:17:42,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41847-0x1012028b2df0000, quorum=127.0.0.1:49767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T09:17:42,144 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4766658a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T09:17:42,147 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@320542b5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T09:17:42,147 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T09:17:42,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39bf03c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T09:17:42,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29886710{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/hadoop.log.dir/,STOPPED} 2024-11-09T09:17:42,149 WARN [BP-994757153-172.17.0.2-1731143599737 heartbeating to localhost/127.0.0.1:39925 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T09:17:42,149 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T09:17:42,149 WARN [BP-994757153-172.17.0.2-1731143599737 heartbeating to localhost/127.0.0.1:39925 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-994757153-172.17.0.2-1731143599737 (Datanode Uuid 4f8332e9-6c27-4623-91d4-6fc003390f36) service to localhost/127.0.0.1:39925 2024-11-09T09:17:42,149 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T09:17:42,150 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/dfs/data/data1/current/BP-994757153-172.17.0.2-1731143599737 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T09:17:42,150 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/cluster_3c847f93-e78b-78d9-aec0-d7dc244992d9/dfs/data/data2/current/BP-994757153-172.17.0.2-1731143599737 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T09:17:42,151 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T09:17:42,155 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@235e3378{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T09:17:42,156 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7481235f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T09:17:42,156 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T09:17:42,156 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79eda29b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T09:17:42,156 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@301bbbb5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ec4e19b2-6f7a-d64c-a46e-796e6214d47b/hadoop.log.dir/,STOPPED} 2024-11-09T09:17:42,165 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-09T09:17:42,199 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down