2023-09-29 06:16:23,402 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 2023-09-29 06:16:23,415 main DEBUG Took 0.010499 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-29 06:16:23,415 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-29 06:16:23,415 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-29 06:16:23,416 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-29 06:16:23,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,426 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-29 06:16:23,437 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,439 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,440 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,441 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,442 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,443 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,444 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,445 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,445 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,446 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,447 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:16:23,447 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,448 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-29 06:16:23,449 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:16:23,451 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-29 06:16:23,453 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-29 06:16:23,453 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-29 06:16:23,454 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-29 06:16:23,455 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-29 06:16:23,465 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-29 06:16:23,468 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-29 06:16:23,469 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-29 06:16:23,469 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-29 06:16:23,470 main DEBUG createAppenders(={Console}) 2023-09-29 06:16:23,470 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 initialized 2023-09-29 06:16:23,471 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 2023-09-29 06:16:23,471 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 OK. 2023-09-29 06:16:23,472 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-29 06:16:23,472 main DEBUG OutputStream closed 2023-09-29 06:16:23,472 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-29 06:16:23,472 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-29 06:16:23,473 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@1fa268de OK 2023-09-29 06:16:23,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-29 06:16:23,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-29 06:16:23,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-29 06:16:23,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-29 06:16:23,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-29 06:16:23,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-29 06:16:23,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-29 06:16:23,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-29 06:16:23,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-29 06:16:23,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-29 06:16:23,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-29 06:16:23,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-29 06:16:23,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-29 06:16:23,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-29 06:16:23,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-29 06:16:23,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-29 06:16:23,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-29 06:16:23,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-29 06:16:23,534 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-29 06:16:23,534 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7205765b) with optional ClassLoader: null 2023-09-29 06:16:23,537 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-29 06:16:23,538 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@7205765b] started OK. 2023-09-29T06:16:23,737 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2023-09-29 06:16:23,740 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-29 06:16:23,740 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-29T06:16:23,827 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000 2023-09-29T06:16:23,855 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-29T06:16:23,859 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/cluster_77b02200-3a07-81cb-6660-8009703e16e4, deleteOnExit=true 2023-09-29T06:16:23,859 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-29T06:16:23,860 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/test.cache.data in system properties and HBase conf 2023-09-29T06:16:23,860 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/hadoop.tmp.dir in system properties and HBase conf 2023-09-29T06:16:23,860 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/hadoop.log.dir in system properties and HBase conf 2023-09-29T06:16:23,861 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-29T06:16:23,861 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-29T06:16:23,861 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-29T06:16:23,953 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-29T06:16:24,312 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-29T06:16:24,315 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-29T06:16:24,316 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-29T06:16:24,316 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-29T06:16:24,316 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-29T06:16:24,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-29T06:16:24,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-29T06:16:24,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-29T06:16:24,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-29T06:16:24,318 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-29T06:16:24,318 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/nfs.dump.dir in system properties and HBase conf 2023-09-29T06:16:24,318 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/java.io.tmpdir in system properties and HBase conf 2023-09-29T06:16:24,319 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-29T06:16:24,319 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-29T06:16:24,319 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-29T06:16:24,724 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-29T06:16:24,728 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-29T06:16:24,990 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-29T06:16:25,125 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@70903d21 via org.mortbay.log.Slf4jLog 2023-09-29T06:16:25,147 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-29T06:16:25,201 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-29T06:16:25,246 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/java.io.tmpdir/Jetty_localhost_33197_hdfs____owkex/webapp 2023-09-29T06:16:25,405 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33197 2023-09-29T06:16:25,435 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-29T06:16:25,436 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-29T06:16:25,967 WARN [Listener at localhost/45753 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-29T06:16:26,030 WARN [Listener at localhost/45753 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-29T06:16:26,047 WARN [Listener at localhost/45753 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-29T06:16:26,053 INFO [Listener at localhost/45753 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-29T06:16:26,058 INFO [Listener at localhost/45753 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/java.io.tmpdir/Jetty_localhost_33707_datanode____yyf6dh/webapp 2023-09-29T06:16:26,161 INFO [Listener at localhost/45753 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33707 2023-09-29T06:16:26,355 WARN [Listener at localhost/43419 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-29T06:16:26,623 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-29T06:16:26,765 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x8cdb6e32e93e2b3e: Processing first storage report for DS-46b5b8b0-6812-437a-bf57-24335aa2da0b from datanode 2e3782e2-0094-4bd8-b140-3621a96bf897 2023-09-29T06:16:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x8cdb6e32e93e2b3e: from storage DS-46b5b8b0-6812-437a-bf57-24335aa2da0b node DatanodeRegistration(127.0.0.1:35953, datanodeUuid=2e3782e2-0094-4bd8-b140-3621a96bf897, infoPort=39637, infoSecurePort=0, ipcPort=43419, storageInfo=lv=-57;cid=testClusterID;nsid=862339001;c=1695968184797), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-29T06:16:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x8cdb6e32e93e2b3e: Processing first storage report for DS-1d693a01-3486-4924-9175-ee9ddde2d592 from datanode 2e3782e2-0094-4bd8-b140-3621a96bf897 2023-09-29T06:16:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x8cdb6e32e93e2b3e: from storage DS-1d693a01-3486-4924-9175-ee9ddde2d592 node DatanodeRegistration(127.0.0.1:35953, datanodeUuid=2e3782e2-0094-4bd8-b140-3621a96bf897, infoPort=39637, infoSecurePort=0, ipcPort=43419, storageInfo=lv=-57;cid=testClusterID;nsid=862339001;c=1695968184797), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-29T06:16:26,848 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000 2023-09-29T06:16:26,913 INFO [Listener at localhost/43419 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/cluster_77b02200-3a07-81cb-6660-8009703e16e4/zookeeper_0, clientPort=62866, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/cluster_77b02200-3a07-81cb-6660-8009703e16e4/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/cluster_77b02200-3a07-81cb-6660-8009703e16e4/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-29T06:16:26,928 INFO [Listener at localhost/43419 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=62866 2023-09-29T06:16:26,935 INFO [Listener at localhost/43419 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:26,937 INFO [Listener at localhost/43419 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:27,620 INFO [Listener at localhost/43419 {}] util.FSUtils(462): Created version file at hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 with version=8 2023-09-29T06:16:27,620 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/hbase-staging 2023-09-29T06:16:27,806 DEBUG [Listener at localhost/43419 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-29T06:16:28,409 INFO [Listener at localhost/43419 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=9 2023-09-29T06:16:28,442 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:16:28,443 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-29T06:16:28,443 INFO [Listener at localhost/43419 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-29T06:16:28,443 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:16:28,443 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-29T06:16:28,586 INFO [Listener at localhost/43419 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-29T06:16:28,676 INFO [Listener at localhost/43419 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-29T06:16:28,692 DEBUG [Listener at localhost/43419 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-29T06:16:28,697 INFO [Listener at localhost/43419 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-29T06:16:28,746 DEBUG [Listener at localhost/43419 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 24749 (auto-detected) 2023-09-29T06:16:28,748 DEBUG [Listener at localhost/43419 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-29T06:16:28,785 INFO [Listener at localhost/43419 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:38609 2023-09-29T06:16:28,797 INFO [Listener at localhost/43419 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:28,799 INFO [Listener at localhost/43419 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:28,817 INFO [Listener at localhost/43419 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:38609 connecting to ZooKeeper ensemble=127.0.0.1:62866 2023-09-29T06:16:28,857 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:386090x0, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-29T06:16:28,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:38609-0x101625e326e0000 connected 2023-09-29T06:16:28,885 DEBUG [Listener at localhost/43419 {}] zookeeper.ZKUtil(113): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-29T06:16:28,887 DEBUG [Listener at localhost/43419 {}] zookeeper.ZKUtil(113): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:16:28,891 DEBUG [Listener at localhost/43419 {}] zookeeper.ZKUtil(113): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-09-29T06:16:28,898 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38609 2023-09-29T06:16:28,898 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38609 2023-09-29T06:16:28,898 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38609 2023-09-29T06:16:28,899 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38609 2023-09-29T06:16:28,899 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38609 2023-09-29T06:16:28,909 INFO [Listener at localhost/43419 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606, hbase.cluster.distributed=false 2023-09-29T06:16:28,965 INFO [Listener at localhost/43419 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=9 2023-09-29T06:16:28,965 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:16:28,965 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-29T06:16:28,965 INFO [Listener at localhost/43419 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-29T06:16:28,966 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:16:28,966 INFO [Listener at localhost/43419 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-29T06:16:28,970 INFO [Listener at localhost/43419 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-29T06:16:28,972 INFO [Listener at localhost/43419 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-29T06:16:28,973 INFO [Listener at localhost/43419 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:43651 2023-09-29T06:16:28,975 INFO [Listener at localhost/43419 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-29T06:16:28,982 DEBUG [Listener at localhost/43419 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-29T06:16:28,982 INFO [Listener at localhost/43419 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:28,985 INFO [Listener at localhost/43419 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:28,987 INFO [Listener at localhost/43419 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:43651 connecting to ZooKeeper ensemble=127.0.0.1:62866 2023-09-29T06:16:28,990 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:436510x0, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-29T06:16:28,991 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:43651-0x101625e326e0001 connected 2023-09-29T06:16:28,992 DEBUG [Listener at localhost/43419 {}] zookeeper.ZKUtil(113): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-29T06:16:28,993 DEBUG [Listener at localhost/43419 {}] zookeeper.ZKUtil(113): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:16:28,994 DEBUG [Listener at localhost/43419 {}] zookeeper.ZKUtil(113): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-09-29T06:16:28,994 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43651 2023-09-29T06:16:28,995 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43651 2023-09-29T06:16:28,995 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43651 2023-09-29T06:16:28,995 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43651 2023-09-29T06:16:28,996 DEBUG [Listener at localhost/43419 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43651 2023-09-29T06:16:28,998 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /hbase/backup-masters/jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:29,006 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:16:29,006 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:16:29,008 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:29,021 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:38609 2023-09-29T06:16:29,027 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-09-29T06:16:29,027 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-09-29T06:16:29,028 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:29,028 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:29,029 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-09-29T06:16:29,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-09-29T06:16:29,030 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/jenkins-hbase3.apache.org,38609,1695968187790 from backup master directory 2023-09-29T06:16:29,033 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:29,033 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:16:29,033 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:16:29,034 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-29T06:16:29,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:29,037 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-29T06:16:29,038 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-29T06:16:29,113 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/hbase.id with ID: 1041713e-1ff7-4d44-bab0-183acee793dc 2023-09-29T06:16:29,152 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:16:29,165 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:29,165 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:29,204 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:16:29,206 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-29T06:16:29,222 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-29T06:16:29,222 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-29T06:16:29,223 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:16:29,231 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:16:29,233 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-29T06:16:29,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store 2023-09-29T06:16:29,715 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-29T06:16:29,716 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:16:29,716 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-29T06:16:29,716 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:29,717 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:29,717 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-29T06:16:29,717 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:29,717 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:29,717 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-29T06:16:29,719 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/.initializing 2023-09-29T06:16:29,719 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/WALs/jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:29,729 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-29T06:16:29,742 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38609%2C1695968187790, suffix=, logDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/WALs/jenkins-hbase3.apache.org,38609,1695968187790, archiveDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/oldWALs, maxLogs=10 2023-09-29T06:16:29,768 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/WALs/jenkins-hbase3.apache.org,38609,1695968187790/jenkins-hbase3.apache.org%2C38609%2C1695968187790.1695968189749, exclude list is [], retry=0 2023-09-29T06:16:29,794 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35953,DS-46b5b8b0-6812-437a-bf57-24335aa2da0b,DISK] 2023-09-29T06:16:29,801 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:16:29,866 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/WALs/jenkins-hbase3.apache.org,38609,1695968187790/jenkins-hbase3.apache.org%2C38609%2C1695968187790.1695968189749 2023-09-29T06:16:29,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35953,DS-46b5b8b0-6812-437a-bf57-24335aa2da0b,DISK]] 2023-09-29T06:16:29,867 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:16:29,867 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:16:29,871 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,872 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-29T06:16:29,959 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:29,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:16:29,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-29T06:16:29,966 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:29,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:16:29,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-29T06:16:29,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:29,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:16:29,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-29T06:16:29,975 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:29,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:16:29,980 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,981 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:29,993 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-29T06:16:29,996 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:16:30,000 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:16:30,001 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67163712, jitterRate=8.172988891601562E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-29T06:16:30,008 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-29T06:16:30,009 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-29T06:16:30,049 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2999a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:16:30,096 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-29T06:16:30,112 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-29T06:16:30,112 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-29T06:16:30,115 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-29T06:16:30,116 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-29T06:16:30,122 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 5 msec 2023-09-29T06:16:30,122 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-29T06:16:30,152 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-29T06:16:30,167 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2023-09-29T06:16:30,169 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/balancer already deleted, retry=false 2023-09-29T06:16:30,172 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-29T06:16:30,173 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2023-09-29T06:16:30,175 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/normalizer already deleted, retry=false 2023-09-29T06:16:30,177 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-29T06:16:30,181 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2023-09-29T06:16:30,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/split already deleted, retry=false 2023-09-29T06:16:30,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2023-09-29T06:16:30,186 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/merge already deleted, retry=false 2023-09-29T06:16:30,198 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-29T06:16:30,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/snapshot-cleanup already deleted, retry=false 2023-09-29T06:16:30,205 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-09-29T06:16:30,205 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-09-29T06:16:30,205 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:30,205 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:30,205 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,38609,1695968187790, sessionid=0x101625e326e0000, setting cluster-up flag (Was=false) 2023-09-29T06:16:30,220 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:30,220 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:30,225 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2023-09-29T06:16:30,227 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:30,237 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:30,237 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:30,242 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2023-09-29T06:16:30,243 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:16:30,309 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:43651 2023-09-29T06:16:30,311 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(997): ClusterId : 1041713e-1ff7-4d44-bab0-183acee793dc 2023-09-29T06:16:30,315 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-29T06:16:30,321 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-29T06:16:30,321 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-29T06:16:30,325 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-29T06:16:30,326 DEBUG [RS:0;jenkins-hbase3:43651 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779e4931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:16:30,328 DEBUG [RS:0;jenkins-hbase3:43651 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32392e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-29T06:16:30,334 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-29T06:16:30,334 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-29T06:16:30,334 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-29T06:16:30,337 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,38609,1695968187790 with isa=jenkins-hbase3.apache.org/172.31.12.81:43651, startcode=1695968188964 2023-09-29T06:16:30,354 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-29T06:16:30,361 DEBUG [RS:0;jenkins-hbase3:43651 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-29T06:16:30,361 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-29T06:16:30,365 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-29T06:16:30,373 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,38609,1695968187790 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-29T06:16:30,380 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:16:30,383 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1695968220383 2023-09-29T06:16:30,385 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-29T06:16:30,393 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-29T06:16:30,393 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-29T06:16:30,396 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-29T06:16:30,400 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:30,400 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-29T06:16:30,404 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-29T06:16:30,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-29T06:16:30,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-29T06:16:30,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-29T06:16:30,407 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-29T06:16:30,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-29T06:16:30,410 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-29T06:16:30,411 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-29T06:16:30,414 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-29T06:16:30,415 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-29T06:16:30,415 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50931, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-29T06:16:30,419 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1695968190418,5,FailOnTimeoutGroup] 2023-09-29T06:16:30,422 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1695968190419,5,FailOnTimeoutGroup] 2023-09-29T06:16:30,426 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-29T06:16:30,427 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-29T06:16:30,427 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38609 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:16:30,427 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-29T06:16:30,428 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:16:30,428 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-29T06:16:30,428 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-29T06:16:30,457 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-29T06:16:30,457 WARN [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 30000 ms and then retrying. 2023-09-29T06:16:30,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:16:30,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-29T06:16:30,848 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-29T06:16:30,848 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:30,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:16:30,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-29T06:16:30,851 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-29T06:16:30,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:30,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:16:30,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-29T06:16:30,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-29T06:16:30,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:16:30,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:16:30,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740 2023-09-29T06:16:30,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740 2023-09-29T06:16:30,860 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-29T06:16:30,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-29T06:16:30,865 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:16:30,866 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72569776, jitterRate=0.08137392997741699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-29T06:16:30,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-29T06:16:30,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-29T06:16:30,868 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-29T06:16:30,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-29T06:16:30,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-29T06:16:30,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-29T06:16:30,870 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-29T06:16:30,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-29T06:16:30,873 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-29T06:16:30,873 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-29T06:16:30,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-29T06:16:30,898 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-29T06:16:30,901 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-29T06:16:31,052 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:32,053 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:33,053 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:34,054 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:35,055 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:36,055 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:37,056 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:38,056 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:38,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-29T06:16:38,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-29T06:16:39,057 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:40,057 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:41,058 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:42,058 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:43,059 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:44,060 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:45,060 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:46,061 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:47,061 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:48,062 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:49,062 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:50,063 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:51,063 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:52,064 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:53,064 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:54,065 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:55,066 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:56,066 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:57,067 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:58,067 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:16:59,068 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:00,068 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:00,331 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-29T06:17:00,337 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43364, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-29T06:17:00,459 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,38609,1695968187790 with isa=jenkins-hbase3.apache.org/172.31.12.81:43651, startcode=1695968188964 2023-09-29T06:17:00,464 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38609 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:00,473 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:00,473 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:45753 2023-09-29T06:17:00,473 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=-1 2023-09-29T06:17:00,478 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-09-29T06:17:00,479 DEBUG [RS:0;jenkins-hbase3:43651 {}] zookeeper.ZKUtil(111): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:00,479 WARN [RS:0;jenkins-hbase3:43651 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-29T06:17:00,479 INFO [RS:0;jenkins-hbase3:43651 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-29T06:17:00,479 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:00,482 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,43651,1695968188964] 2023-09-29T06:17:00,496 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-29T06:17:00,509 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-29T06:17:00,520 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-29T06:17:00,523 INFO [RS:0;jenkins-hbase3:43651 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-29T06:17:00,524 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,524 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-29T06:17:00,535 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,535 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,535 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,535 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,535 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,536 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,536 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-29T06:17:00,536 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,536 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,537 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,537 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,537 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:17:00,537 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-29T06:17:00,537 DEBUG [RS:0;jenkins-hbase3:43651 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-29T06:17:00,538 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,539 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,539 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,539 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,539 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43651,1695968188964-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-29T06:17:00,559 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-29T06:17:00,561 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43651,1695968188964-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:00,576 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,43651,1695968188964 started 2023-09-29T06:17:00,576 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,43651,1695968188964, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:43651, sessionid=0x101625e326e0001 2023-09-29T06:17:00,577 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-29T06:17:00,577 DEBUG [RS:0;jenkins-hbase3:43651 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:00,577 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,43651,1695968188964' 2023-09-29T06:17:00,577 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2023-09-29T06:17:00,578 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2023-09-29T06:17:00,578 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-29T06:17:00,579 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-29T06:17:00,579 DEBUG [RS:0;jenkins-hbase3:43651 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:00,579 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,43651,1695968188964' 2023-09-29T06:17:00,579 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2023-09-29T06:17:00,579 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2023-09-29T06:17:00,580 DEBUG [RS:0;jenkins-hbase3:43651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-29T06:17:00,580 INFO [RS:0;jenkins-hbase3:43651 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-29T06:17:00,580 INFO [RS:0;jenkins-hbase3:43651 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-29T06:17:01,069 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:02,069 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:03,070 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:04,071 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:05,071 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:06,072 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:07,072 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:08,073 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:09,073 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:10,074 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:11,075 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:12,075 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:13,076 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:14,076 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:15,077 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:16,077 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:17,078 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:18,078 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:19,079 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:20,079 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:21,080 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:22,080 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:23,081 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:24,082 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:25,082 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:26,083 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:27,083 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:28,084 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:29,084 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:30,085 WARN [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:17:30,588 INFO [RS:0;jenkins-hbase3:43651 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-29T06:17:30,593 INFO [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43651%2C1695968188964, suffix=, logDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964, archiveDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/oldWALs, maxLogs=32 2023-09-29T06:17:30,606 DEBUG [RS:0;jenkins-hbase3:43651 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964/jenkins-hbase3.apache.org%2C43651%2C1695968188964.1695968250595, exclude list is [], retry=0 2023-09-29T06:17:30,611 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35953,DS-46b5b8b0-6812-437a-bf57-24335aa2da0b,DISK] 2023-09-29T06:17:30,616 INFO [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964/jenkins-hbase3.apache.org%2C43651%2C1695968188964.1695968250595 2023-09-29T06:17:30,616 DEBUG [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35953,DS-46b5b8b0-6812-437a-bf57-24335aa2da0b,DISK]] 2023-09-29T06:17:30,837 DEBUG [jenkins-hbase3:38609 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-29T06:17:30,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:30,850 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,43651,1695968188964, state=OPENING 2023-09-29T06:17:30,857 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-29T06:17:30,859 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:17:30,859 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:17:30,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:17:30,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:17:30,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:31,048 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:31,050 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-29T06:17:31,053 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51558, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-29T06:17:31,067 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-29T06:17:31,068 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-29T06:17:31,068 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-29T06:17:31,071 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43651%2C1695968188964.meta, suffix=.meta, logDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964, archiveDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/oldWALs, maxLogs=32 2023-09-29T06:17:31,082 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964/jenkins-hbase3.apache.org%2C43651%2C1695968188964.meta.1695968251072.meta, exclude list is [], retry=0 2023-09-29T06:17:31,086 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35953,DS-46b5b8b0-6812-437a-bf57-24335aa2da0b,DISK] 2023-09-29T06:17:31,091 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/WALs/jenkins-hbase3.apache.org,43651,1695968188964/jenkins-hbase3.apache.org%2C43651%2C1695968188964.meta.1695968251072.meta 2023-09-29T06:17:31,091 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35953,DS-46b5b8b0-6812-437a-bf57-24335aa2da0b,DISK]] 2023-09-29T06:17:31,091 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:17:31,093 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-29T06:17:31,109 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-29T06:17:31,116 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-29T06:17:31,122 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-29T06:17:31,122 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:31,122 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-29T06:17:31,122 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-29T06:17:31,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-29T06:17:31,126 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-29T06:17:31,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:31,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:17:31,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-29T06:17:31,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-29T06:17:31,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:31,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:17:31,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-29T06:17:31,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-29T06:17:31,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:31,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:17:31,133 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740 2023-09-29T06:17:31,135 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740 2023-09-29T06:17:31,137 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-29T06:17:31,140 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-29T06:17:31,141 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69858253, jitterRate=0.040969088673591614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-29T06:17:31,143 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-29T06:17:31,155 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1695968251040 2023-09-29T06:17:31,172 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-29T06:17:31,173 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-29T06:17:31,174 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:31,175 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,43651,1695968188964, state=OPEN 2023-09-29T06:17:31,178 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-09-29T06:17:31,178 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-09-29T06:17:31,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:17:31,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:17:31,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-29T06:17:31,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,43651,1695968188964 in 316 msec 2023-09-29T06:17:31,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-29T06:17:31,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.305 sec 2023-09-29T06:17:31,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1 mins, 0.905 sec 2023-09-29T06:17:31,196 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1695968251196, completionTime=-1 2023-09-29T06:17:31,196 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-29T06:17:31,196 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-29T06:17:31,251 DEBUG [hconnection-0x22d319da-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:31,253 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51566, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:31,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-29T06:17:31,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1695968311266 2023-09-29T06:17:31,267 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1695968371267 2023-09-29T06:17:31,267 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 70 msec 2023-09-29T06:17:31,291 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38609,1695968187790-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:31,291 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38609,1695968187790-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:31,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38609,1695968187790-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:31,293 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:38609, period=300000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:31,294 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-29T06:17:31,299 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-29T06:17:31,307 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-29T06:17:31,308 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-29T06:17:31,316 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-29T06:17:31,319 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:17:31,321 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:31,322 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:17:31,740 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 144c806451470faf2697726823a29c67, NAME => 'hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:31,751 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:31,752 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 144c806451470faf2697726823a29c67, disabling compactions & flushes 2023-09-29T06:17:31,752 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:31,752 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:31,752 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. after waiting 0 ms 2023-09-29T06:17:31,752 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:31,752 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:31,752 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 144c806451470faf2697726823a29c67: 2023-09-29T06:17:31,754 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:17:31,765 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1695968251756"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968251756"}]},"ts":"1695968251756"} 2023-09-29T06:17:31,794 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:17:31,795 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:17:31,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968251796"}]},"ts":"1695968251796"} 2023-09-29T06:17:31,803 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-29T06:17:31,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=144c806451470faf2697726823a29c67, ASSIGN}] 2023-09-29T06:17:31,811 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=144c806451470faf2697726823a29c67, ASSIGN 2023-09-29T06:17:31,813 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=144c806451470faf2697726823a29c67, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:17:31,964 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=144c806451470faf2697726823a29c67, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:31,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 144c806451470faf2697726823a29c67, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:32,128 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:32,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 144c806451470faf2697726823a29c67, NAME => 'hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:17:32,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 144c806451470faf2697726823a29c67 2023-09-29T06:17:32,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:32,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 144c806451470faf2697726823a29c67 2023-09-29T06:17:32,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 144c806451470faf2697726823a29c67 2023-09-29T06:17:32,131 INFO [StoreOpener-144c806451470faf2697726823a29c67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 144c806451470faf2697726823a29c67 2023-09-29T06:17:32,133 INFO [StoreOpener-144c806451470faf2697726823a29c67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 144c806451470faf2697726823a29c67 columnFamilyName info 2023-09-29T06:17:32,133 DEBUG [StoreOpener-144c806451470faf2697726823a29c67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:32,134 INFO [StoreOpener-144c806451470faf2697726823a29c67-1 {}] regionserver.HStore(324): Store=144c806451470faf2697726823a29c67/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:17:32,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67 2023-09-29T06:17:32,136 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67 2023-09-29T06:17:32,139 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 144c806451470faf2697726823a29c67 2023-09-29T06:17:32,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:17:32,143 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 144c806451470faf2697726823a29c67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64915044, jitterRate=-0.032690465450286865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:17:32,144 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 144c806451470faf2697726823a29c67: 2023-09-29T06:17:32,146 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67., pid=6, masterSystemTime=1695968252122 2023-09-29T06:17:32,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:32,150 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:17:32,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=144c806451470faf2697726823a29c67, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:32,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-29T06:17:32,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 144c806451470faf2697726823a29c67, server=jenkins-hbase3.apache.org,43651,1695968188964 in 187 msec 2023-09-29T06:17:32,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-29T06:17:32,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=144c806451470faf2697726823a29c67, ASSIGN in 349 msec 2023-09-29T06:17:32,163 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:17:32,163 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968252163"}]},"ts":"1695968252163"} 2023-09-29T06:17:32,166 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-29T06:17:32,169 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:17:32,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 860 msec 2023-09-29T06:17:32,219 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2023-09-29T06:17:32,221 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2023-09-29T06:17:32,221 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:17:32,221 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:17:32,260 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-29T06:17:32,283 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-09-29T06:17:32,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 34 msec 2023-09-29T06:17:32,294 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-29T06:17:32,306 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-09-29T06:17:32,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-09-29T06:17:32,322 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2023-09-29T06:17:32,324 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2023-09-29T06:17:32,324 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 63.289sec 2023-09-29T06:17:32,326 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-29T06:17:32,328 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-29T06:17:32,329 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-29T06:17:32,329 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-29T06:17:32,329 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-29T06:17:32,331 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38609,1695968187790-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-29T06:17:32,331 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38609,1695968187790-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-29T06:17:32,341 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-29T06:17:32,343 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-29T06:17:32,355 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x606f5fd4 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:32,363 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b791d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:32,366 DEBUG [Listener at localhost/43419 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-29T06:17:32,367 DEBUG [Listener at localhost/43419 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-29T06:17:32,380 DEBUG [hconnection-0x5f04a62e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:32,390 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:32,401 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:17:32,420 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=195, OpenFileDescriptor=480, MaxFileDescriptor=60000, SystemLoadAverage=102, ProcessCount=167, AvailableMemoryMB=1992 2023-09-29T06:17:32,424 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x141f47c6 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:32,436 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x141f47c6 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3107136f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:32,440 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x01315317 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:32,444 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ba863d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:32,453 DEBUG [Listener at localhost/43419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:17:32,456 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:17:32,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:17:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2023-09-29T06:17:32,473 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:17:32,473 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:32,475 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:17:32,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2023-09-29T06:17:32,491 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d2ca20f2151278b7b07397ded57f313b, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:17:32,904 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:32,904 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing d2ca20f2151278b7b07397ded57f313b, disabling compactions & flushes 2023-09-29T06:17:32,904 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:32,904 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:32,905 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. after waiting 0 ms 2023-09-29T06:17:32,905 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:32,905 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:32,905 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for d2ca20f2151278b7b07397ded57f313b: 2023-09-29T06:17:32,907 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:17:32,908 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1695968252908"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968252908"}]},"ts":"1695968252908"} 2023-09-29T06:17:32,912 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:17:32,913 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:17:32,914 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968252913"}]},"ts":"1695968252913"} 2023-09-29T06:17:32,917 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:17:32,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=d2ca20f2151278b7b07397ded57f313b, ASSIGN}] 2023-09-29T06:17:32,923 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=d2ca20f2151278b7b07397ded57f313b, ASSIGN 2023-09-29T06:17:32,924 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=d2ca20f2151278b7b07397ded57f313b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:17:33,075 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=d2ca20f2151278b7b07397ded57f313b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:33,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure d2ca20f2151278b7b07397ded57f313b, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:33,239 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:33,240 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7254): Opening region: {ENCODED => d2ca20f2151278b7b07397ded57f313b, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:17:33,240 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,240 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:33,240 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7296): checking encryption for d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,240 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7299): checking classloading for d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,242 INFO [StoreOpener-d2ca20f2151278b7b07397ded57f313b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,244 INFO [StoreOpener-d2ca20f2151278b7b07397ded57f313b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2ca20f2151278b7b07397ded57f313b columnFamilyName testFamily 2023-09-29T06:17:33,244 DEBUG [StoreOpener-d2ca20f2151278b7b07397ded57f313b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:33,245 INFO [StoreOpener-d2ca20f2151278b7b07397ded57f313b-1 {}] regionserver.HStore(324): Store=d2ca20f2151278b7b07397ded57f313b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:17:33,247 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,248 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,252 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1076): writing seq id for d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:17:33,255 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:17:33,256 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): Opened d2ca20f2151278b7b07397ded57f313b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59716257, jitterRate=-0.11015842854976654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:17:33,258 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(992): Region open journal for d2ca20f2151278b7b07397ded57f313b: 2023-09-29T06:17:33,259 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b., pid=11, masterSystemTime=1695968253233 2023-09-29T06:17:33,263 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:33,263 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:17:33,264 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=d2ca20f2151278b7b07397ded57f313b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:33,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=11, resume processing ppid=10 2023-09-29T06:17:33,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure d2ca20f2151278b7b07397ded57f313b, server=jenkins-hbase3.apache.org,43651,1695968188964 in 189 msec 2023-09-29T06:17:33,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=10, resume processing ppid=9 2023-09-29T06:17:33,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=d2ca20f2151278b7b07397ded57f313b, ASSIGN in 350 msec 2023-09-29T06:17:33,276 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:17:33,276 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968253276"}]},"ts":"1695968253276"} 2023-09-29T06:17:33,279 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:17:33,282 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:17:33,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 814 msec 2023-09-29T06:17:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:17:33,501 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2023-09-29T06:17:33,501 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:17:33,502 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:33,508 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2023-09-29T06:17:33,508 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:33,508 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2023-09-29T06:17:33,511 DEBUG [hconnection-0x7238f38e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:33,513 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51578, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:33,527 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:17:33,532 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-09-29T06:17:33,549 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:17:33,551 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x141f47c6 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:17:33,560 DEBUG [Listener at localhost/43419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:33,563 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51594, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:17:33,574 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-09-29T06:17:33,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032971 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:33,577 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-09-29T06:17:33,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032971 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:33,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101032971 2023-09-29T06:17:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:51594 deadline: 1695968256576, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:33,580 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032971 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:17:33,583 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-09-29T06:17:33,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032972 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:33,584 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-09-29T06:17:33,584 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:33,586 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:33,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032973 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:33,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101032973 2023-09-29T06:17:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:51594 deadline: 1695968256585, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:33,587 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032973 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:17:33,589 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:33,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032974 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:33,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4693677206101032974 2023-09-29T06:17:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:51594 deadline: 1695968256588, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:33,592 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:33,590 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032974 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:17:33,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032975 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:33,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4693677206101032975 2023-09-29T06:17:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:51594 deadline: 1695968256591, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:33,595 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032975 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:17:33,596 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:33,597 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x01315317 to 127.0.0.1:62866 2023-09-29T06:17:33,597 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:33,598 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:17:33,598 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:17:33,606 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:33,606 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x141f47c6 to 127.0.0.1:62866 2023-09-29T06:17:33,620 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=209 (was 195) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62866@0x141f47c6 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$438/1940435785.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1278504011_18 at /127.0.0.1:42816 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x5f04a62e-shared-pool-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RSProcedureDispatcher-pool-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.lang.Thread.sleep(Native Method) org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:600) org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:496) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: process reaper sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62866@0x141f47c6-SendThread(127.0.0.1:62866) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62866@0x01315317 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$438/1940435785.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62866@0x01315317-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62866@0x01315317-SendThread(127.0.0.1:62866) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62866@0x141f47c6-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) - Thread LEAK? -, OpenFileDescriptor=492 (was 480) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=102 (was 102), ProcessCount=167 (was 167), AvailableMemoryMB=1972 (was 1992) 2023-09-29T06:17:33,630 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=209, OpenFileDescriptor=492, MaxFileDescriptor=60000, SystemLoadAverage=102, ProcessCount=167, AvailableMemoryMB=1971 2023-09-29T06:17:33,632 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1bfdfba4 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:33,637 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x1bfdfba4 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42263159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:33,638 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x644ca7ef to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:33,643 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9663a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:33,646 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:17:33,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2023-09-29T06:17:33,650 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:17:33,650 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:33,650 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2023-09-29T06:17:33,651 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:17:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:17:33,665 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3b2e2b8af1b0a36351d7765af650b765, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:34,078 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:34,078 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 3b2e2b8af1b0a36351d7765af650b765, disabling compactions & flushes 2023-09-29T06:17:34,078 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,078 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,078 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. after waiting 0 ms 2023-09-29T06:17:34,078 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,078 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,078 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3b2e2b8af1b0a36351d7765af650b765: 2023-09-29T06:17:34,081 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:17:34,081 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1695968254081"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968254081"}]},"ts":"1695968254081"} 2023-09-29T06:17:34,084 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:17:34,086 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:17:34,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968254086"}]},"ts":"1695968254086"} 2023-09-29T06:17:34,088 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:17:34,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=3b2e2b8af1b0a36351d7765af650b765, ASSIGN}] 2023-09-29T06:17:34,095 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=3b2e2b8af1b0a36351d7765af650b765, ASSIGN 2023-09-29T06:17:34,096 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=3b2e2b8af1b0a36351d7765af650b765, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:17:34,247 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=3b2e2b8af1b0a36351d7765af650b765, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:34,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 3b2e2b8af1b0a36351d7765af650b765, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:34,408 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,408 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7254): Opening region: {ENCODED => 3b2e2b8af1b0a36351d7765af650b765, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:17:34,409 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,409 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:34,409 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7296): checking encryption for 3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,409 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7299): checking classloading for 3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,411 INFO [StoreOpener-3b2e2b8af1b0a36351d7765af650b765-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,413 INFO [StoreOpener-3b2e2b8af1b0a36351d7765af650b765-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b2e2b8af1b0a36351d7765af650b765 columnFamilyName testFamily 2023-09-29T06:17:34,413 DEBUG [StoreOpener-3b2e2b8af1b0a36351d7765af650b765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:34,414 INFO [StoreOpener-3b2e2b8af1b0a36351d7765af650b765-1 {}] regionserver.HStore(324): Store=3b2e2b8af1b0a36351d7765af650b765/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:17:34,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,419 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1076): writing seq id for 3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:17:34,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:17:34,423 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1093): Opened 3b2e2b8af1b0a36351d7765af650b765; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58792308, jitterRate=-0.12392634153366089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:17:34,425 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(992): Region open journal for 3b2e2b8af1b0a36351d7765af650b765: 2023-09-29T06:17:34,426 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765., pid=14, masterSystemTime=1695968254403 2023-09-29T06:17:34,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,429 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:17:34,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=3b2e2b8af1b0a36351d7765af650b765, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:34,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-09-29T06:17:34,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 3b2e2b8af1b0a36351d7765af650b765, server=jenkins-hbase3.apache.org,43651,1695968188964 in 182 msec 2023-09-29T06:17:34,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=13, resume processing ppid=12 2023-09-29T06:17:34,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=3b2e2b8af1b0a36351d7765af650b765, ASSIGN in 342 msec 2023-09-29T06:17:34,439 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:17:34,440 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968254440"}]},"ts":"1695968254440"} 2023-09-29T06:17:34,442 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:17:34,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:17:34,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 798 msec 2023-09-29T06:17:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:17:34,655 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2023-09-29T06:17:34,655 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:17:34,655 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:34,659 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2023-09-29T06:17:34,659 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:34,659 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2023-09-29T06:17:34,662 DEBUG [hconnection-0x3e86f9e8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:34,664 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51600, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:34,677 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:17:34,680 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:17:34,681 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:17:36,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:17:36,696 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2023-09-29T06:17:36,697 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-29T06:17:36,698 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2023-09-29T06:17:36,699 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-29T06:17:38,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2023-09-29T06:17:38,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:17:38,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-29T06:17:38,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-29T06:17:38,675 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-29T06:17:38,675 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-29T06:17:38,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-29T06:17:38,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-29T06:17:38,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2023-09-29T06:17:38,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:17:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 88 bytes 2023-09-29T06:17:41,715 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:17:43,692 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11029: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11029: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:17:43,693 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-09-29T06:17:43.693Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11029: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11029: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1977ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:17:43,695 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9015 2023-09-29T06:17:43,696 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x644ca7ef to 127.0.0.1:62866 2023-09-29T06:17:43,696 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:43,696 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:17:43,696 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:17:43,696 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:43,696 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1bfdfba4 to 127.0.0.1:62866 2023-09-29T06:17:43,710 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=202 (was 209), OpenFileDescriptor=490 (was 492), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=86 (was 102), ProcessCount=167 (was 167), AvailableMemoryMB=1936 (was 1971) 2023-09-29T06:17:43,721 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=202, OpenFileDescriptor=490, MaxFileDescriptor=60000, SystemLoadAverage=86, ProcessCount=167, AvailableMemoryMB=1934 2023-09-29T06:17:43,723 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7efaf8ed to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:43,728 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x7efaf8ed {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d95ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:43,729 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x34699915 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:43,734 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d24bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:43,737 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:17:43,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2023-09-29T06:17:43,740 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:17:43,740 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:43,740 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2023-09-29T06:17:43,741 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:17:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=15 2023-09-29T06:17:44,158 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 58d56be7906234b32fb9db25b080a032, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:44,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:44,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 58d56be7906234b32fb9db25b080a032, disabling compactions & flushes 2023-09-29T06:17:44,572 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. after waiting 0 ms 2023-09-29T06:17:44,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,572 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 58d56be7906234b32fb9db25b080a032: 2023-09-29T06:17:44,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:17:44,575 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1695968264574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968264574"}]},"ts":"1695968264574"} 2023-09-29T06:17:44,577 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:17:44,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:17:44,578 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968264578"}]},"ts":"1695968264578"} 2023-09-29T06:17:44,580 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:17:44,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=58d56be7906234b32fb9db25b080a032, ASSIGN}] 2023-09-29T06:17:44,586 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=58d56be7906234b32fb9db25b080a032, ASSIGN 2023-09-29T06:17:44,587 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=58d56be7906234b32fb9db25b080a032, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:17:44,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=58d56be7906234b32fb9db25b080a032, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:44,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 58d56be7906234b32fb9db25b080a032, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=15 2023-09-29T06:17:44,903 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7254): Opening region: {ENCODED => 58d56be7906234b32fb9db25b080a032, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:17:44,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:44,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7296): checking encryption for 58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7299): checking classloading for 58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,905 INFO [StoreOpener-58d56be7906234b32fb9db25b080a032-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,907 INFO [StoreOpener-58d56be7906234b32fb9db25b080a032-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58d56be7906234b32fb9db25b080a032 columnFamilyName testFamily 2023-09-29T06:17:44,907 DEBUG [StoreOpener-58d56be7906234b32fb9db25b080a032-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:44,908 INFO [StoreOpener-58d56be7906234b32fb9db25b080a032-1 {}] regionserver.HStore(324): Store=58d56be7906234b32fb9db25b080a032/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:17:44,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,914 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1076): writing seq id for 58d56be7906234b32fb9db25b080a032 2023-09-29T06:17:44,917 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:17:44,918 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1093): Opened 58d56be7906234b32fb9db25b080a032; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72173987, jitterRate=0.07547621428966522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:17:44,919 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(992): Region open journal for 58d56be7906234b32fb9db25b080a032: 2023-09-29T06:17:44,920 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032., pid=17, masterSystemTime=1695968264894 2023-09-29T06:17:44,923 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,923 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:17:44,924 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=58d56be7906234b32fb9db25b080a032, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:44,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-29T06:17:44,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 58d56be7906234b32fb9db25b080a032, server=jenkins-hbase3.apache.org,43651,1695968188964 in 184 msec 2023-09-29T06:17:44,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=16, resume processing ppid=15 2023-09-29T06:17:44,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=58d56be7906234b32fb9db25b080a032, ASSIGN in 344 msec 2023-09-29T06:17:44,933 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:17:44,933 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968264933"}]},"ts":"1695968264933"} 2023-09-29T06:17:44,935 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:17:44,938 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:17:44,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 1.2010 sec 2023-09-29T06:17:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=15 2023-09-29T06:17:46,747 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2023-09-29T06:17:46,747 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:17:46,747 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:46,751 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2023-09-29T06:17:46,751 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:46,752 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2023-09-29T06:17:46,754 DEBUG [hconnection-0x36bbd3bf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:46,757 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46562, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:46,768 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:17:46,771 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:17:46,771 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:17:46,772 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x7efaf8ed {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:17:46,773 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x7efaf8ed {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:46,775 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:17:46,778 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:17:46,779 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032987 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:46,779 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:17:46,780 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032987 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:46,780 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2023-09-29T06:17:48,215 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:17:48,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2023-09-29T06:17:48,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:17:48,701 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2023-09-29T06:17:50,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:17:55,781 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9002 2023-09-29T06:17:55,782 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x34699915 to 127.0.0.1:62866 2023-09-29T06:17:55,782 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032987 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:17:55,782 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:55,782 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:17:55,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101032987' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:17:55,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.31.12.81:46574 deadline: 1695968281781, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101032987' does not exist 2023-09-29T06:17:55,782 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:17:55,782 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:55,783 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7efaf8ed to 127.0.0.1:62866 2023-09-29T06:17:55,784 WARN [RS-EventLoopGroup-3-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /172.31.12.81:46574; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.channel.unix.Errors$NativeIoException: recvAddress(..) failed: Connection reset by peer 2023-09-29T06:17:55,784 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,43651,1695968188964 for closing scanner id = 4693677206101032987 for 58d56be7906234b32fb9db25b080a032 of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-09-29T06:17:55,795 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=202 (was 202), OpenFileDescriptor=483 (was 490), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=73 (was 86), ProcessCount=167 (was 167), AvailableMemoryMB=1889 (was 1934) 2023-09-29T06:17:55,806 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=202, OpenFileDescriptor=483, MaxFileDescriptor=60000, SystemLoadAverage=73, ProcessCount=167, AvailableMemoryMB=1888 2023-09-29T06:17:55,808 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3adf01ed to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:55,848 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x3adf01ed {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3daea7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:55,849 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x32ed33b8 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:55,853 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8188efd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:55,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:17:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2023-09-29T06:17:55,859 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:17:55,859 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:55,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2023-09-29T06:17:55,860 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:17:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:17:56,278 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a5a35f3ea933f9ee2575761d0ca57b8b, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:56,281 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:17:56,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:56,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing a5a35f3ea933f9ee2575761d0ca57b8b, disabling compactions & flushes 2023-09-29T06:17:56,689 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:56,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:56,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. after waiting 0 ms 2023-09-29T06:17:56,690 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:56,690 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:56,690 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for a5a35f3ea933f9ee2575761d0ca57b8b: 2023-09-29T06:17:56,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:17:56,692 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1695968276691"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968276691"}]},"ts":"1695968276691"} 2023-09-29T06:17:56,694 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:17:56,695 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:17:56,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968276695"}]},"ts":"1695968276695"} 2023-09-29T06:17:56,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:17:56,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a5a35f3ea933f9ee2575761d0ca57b8b, ASSIGN}] 2023-09-29T06:17:56,705 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a5a35f3ea933f9ee2575761d0ca57b8b, ASSIGN 2023-09-29T06:17:56,706 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a5a35f3ea933f9ee2575761d0ca57b8b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:17:56,857 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=a5a35f3ea933f9ee2575761d0ca57b8b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:56,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure a5a35f3ea933f9ee2575761d0ca57b8b, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:17:57,019 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:57,020 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7254): Opening region: {ENCODED => a5a35f3ea933f9ee2575761d0ca57b8b, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:17:57,020 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,020 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:57,020 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7296): checking encryption for a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,020 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7299): checking classloading for a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,022 INFO [StoreOpener-a5a35f3ea933f9ee2575761d0ca57b8b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,024 INFO [StoreOpener-a5a35f3ea933f9ee2575761d0ca57b8b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a35f3ea933f9ee2575761d0ca57b8b columnFamilyName testFamily 2023-09-29T06:17:57,024 DEBUG [StoreOpener-a5a35f3ea933f9ee2575761d0ca57b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:57,025 INFO [StoreOpener-a5a35f3ea933f9ee2575761d0ca57b8b-1 {}] regionserver.HStore(324): Store=a5a35f3ea933f9ee2575761d0ca57b8b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:17:57,026 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,027 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,030 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1076): writing seq id for a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:17:57,033 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:17:57,034 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): Opened a5a35f3ea933f9ee2575761d0ca57b8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72388641, jitterRate=0.0786748081445694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:17:57,034 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(992): Region open journal for a5a35f3ea933f9ee2575761d0ca57b8b: 2023-09-29T06:17:57,035 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b., pid=20, masterSystemTime=1695968277012 2023-09-29T06:17:57,038 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:57,038 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:17:57,039 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=a5a35f3ea933f9ee2575761d0ca57b8b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:57,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-09-29T06:17:57,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure a5a35f3ea933f9ee2575761d0ca57b8b, server=jenkins-hbase3.apache.org,43651,1695968188964 in 182 msec 2023-09-29T06:17:57,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=19, resume processing ppid=18 2023-09-29T06:17:57,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a5a35f3ea933f9ee2575761d0ca57b8b, ASSIGN in 340 msec 2023-09-29T06:17:57,047 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:17:57,047 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968277047"}]},"ts":"1695968277047"} 2023-09-29T06:17:57,049 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:17:57,052 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:17:57,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.1960 sec 2023-09-29T06:17:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2023-09-29T06:17:58,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:17:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:17:58,866 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2023-09-29T06:17:58,866 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:17:58,866 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:58,870 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2023-09-29T06:17:58,870 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:17:58,870 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2023-09-29T06:17:58,872 DEBUG [hconnection-0x1cd37493-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:17:58,875 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:17:58,885 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:17:58,888 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-09-29T06:17:58,890 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-09-29T06:17:58,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032991 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:58,892 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-09-29T06:17:58,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032991 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:58,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101032991 2023-09-29T06:17:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:58046 deadline: 1695968279893, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:58,895 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:17:58,898 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-09-29T06:17:58,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032992 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:58,899 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-09-29T06:17:58,899 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:58,901 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:58,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032993 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:58,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101032993 2023-09-29T06:17:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:58046 deadline: 1695968279901, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:58,903 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:17:58,905 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:58,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032994 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:58,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4693677206101032994 2023-09-29T06:17:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:58046 deadline: 1695968279905, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:58,906 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:17:58,908 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:58,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101032995 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:17:58,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4693677206101032995 2023-09-29T06:17:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:58046 deadline: 1695968279909, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:17:58,910 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:17:58,912 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:17:58,912 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x32ed33b8 to 127.0.0.1:62866 2023-09-29T06:17:58,912 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:58,912 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:17:58,913 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:17:58,913 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:17:58,913 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3adf01ed to 127.0.0.1:62866 2023-09-29T06:17:58,923 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=207 (was 202) - Thread LEAK? -, OpenFileDescriptor=489 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=75 (was 73) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=1836 (was 1888) 2023-09-29T06:17:58,933 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=207, OpenFileDescriptor=489, MaxFileDescriptor=60000, SystemLoadAverage=75, ProcessCount=167, AvailableMemoryMB=1835 2023-09-29T06:17:58,935 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x67e1c55f to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:58,939 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x67e1c55f {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281f47d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:58,941 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3eb469e4 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:17:58,944 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd935d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:17:58,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:17:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2023-09-29T06:17:58,949 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:17:58,950 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:17:58,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2023-09-29T06:17:58,951 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:17:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=21 2023-09-29T06:17:59,365 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 881582bf9027dda670e449ba9791bddb, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:17:59,775 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:17:59,775 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 881582bf9027dda670e449ba9791bddb, disabling compactions & flushes 2023-09-29T06:17:59,775 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:17:59,775 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:17:59,775 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. after waiting 0 ms 2023-09-29T06:17:59,775 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:17:59,775 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:17:59,776 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 881582bf9027dda670e449ba9791bddb: 2023-09-29T06:17:59,778 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:17:59,778 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1695968279778"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968279778"}]},"ts":"1695968279778"} 2023-09-29T06:17:59,781 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:17:59,782 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:17:59,782 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968279782"}]},"ts":"1695968279782"} 2023-09-29T06:17:59,784 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:17:59,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=881582bf9027dda670e449ba9791bddb, ASSIGN}] 2023-09-29T06:17:59,794 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=881582bf9027dda670e449ba9791bddb, ASSIGN 2023-09-29T06:17:59,795 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=881582bf9027dda670e449ba9791bddb, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:17:59,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=881582bf9027dda670e449ba9791bddb, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:17:59,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 881582bf9027dda670e449ba9791bddb, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:17:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=21 2023-09-29T06:18:00,106 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:18:00,106 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7254): Opening region: {ENCODED => 881582bf9027dda670e449ba9791bddb, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:00,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:00,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7296): checking encryption for 881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7299): checking classloading for 881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,109 INFO [StoreOpener-881582bf9027dda670e449ba9791bddb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,112 INFO [StoreOpener-881582bf9027dda670e449ba9791bddb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881582bf9027dda670e449ba9791bddb columnFamilyName testFamily 2023-09-29T06:18:00,112 DEBUG [StoreOpener-881582bf9027dda670e449ba9791bddb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:00,113 INFO [StoreOpener-881582bf9027dda670e449ba9791bddb-1 {}] regionserver.HStore(324): Store=881582bf9027dda670e449ba9791bddb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:00,114 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,115 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1076): writing seq id for 881582bf9027dda670e449ba9791bddb 2023-09-29T06:18:00,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:00,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): Opened 881582bf9027dda670e449ba9791bddb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66168992, jitterRate=-0.014005184173583984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:00,122 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(992): Region open journal for 881582bf9027dda670e449ba9791bddb: 2023-09-29T06:18:00,123 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb., pid=23, masterSystemTime=1695968280101 2023-09-29T06:18:00,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:18:00,125 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:18:00,126 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=881582bf9027dda670e449ba9791bddb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:00,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-29T06:18:00,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 881582bf9027dda670e449ba9791bddb, server=jenkins-hbase3.apache.org,43651,1695968188964 in 179 msec 2023-09-29T06:18:00,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=22, resume processing ppid=21 2023-09-29T06:18:00,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=881582bf9027dda670e449ba9791bddb, ASSIGN in 338 msec 2023-09-29T06:18:00,134 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:00,134 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968280134"}]},"ts":"1695968280134"} 2023-09-29T06:18:00,136 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:00,139 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:00,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.1920 sec 2023-09-29T06:18:00,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2023-09-29T06:18:00,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2023-09-29T06:18:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=21 2023-09-29T06:18:01,955 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2023-09-29T06:18:01,955 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:01,956 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:01,959 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:01,960 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:01,960 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2023-09-29T06:18:01,962 DEBUG [hconnection-0x5092704a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:01,964 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43666, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:01,974 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:01,976 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:01,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:18:01,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2023-09-29T06:18:02,548 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 11 bytes 2023-09-29T06:18:03,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:18:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:43666 deadline: 1695968284978, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:18:03,990 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:18:03,991 DEBUG [Listener at localhost/43419 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033000 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:18:03,991 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2014 2023-09-29T06:18:03,992 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3eb469e4 to 127.0.0.1:62866 2023-09-29T06:18:03,992 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:03,992 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:03,992 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:03,992 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:03,992 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x67e1c55f to 127.0.0.1:62866 2023-09-29T06:18:04,004 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=208 (was 207) - Thread LEAK? -, OpenFileDescriptor=492 (was 489) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=69 (was 75), ProcessCount=167 (was 167), AvailableMemoryMB=1834 (was 1835) 2023-09-29T06:18:04,014 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=208, OpenFileDescriptor=492, MaxFileDescriptor=60000, SystemLoadAverage=69, ProcessCount=167, AvailableMemoryMB=1833 2023-09-29T06:18:04,017 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7a8459cd to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:04,021 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x7a8459cd {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0a5bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:04,023 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0da78920 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:04,027 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33751f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:04,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2023-09-29T06:18:04,033 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:04,033 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:04,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2023-09-29T06:18:04,034 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:18:04,449 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b60b1ecdd7b636a2e033c7b1d93d483f, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:04,460 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:04,460 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing b60b1ecdd7b636a2e033c7b1d93d483f, disabling compactions & flushes 2023-09-29T06:18:04,460 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,460 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,460 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. after waiting 0 ms 2023-09-29T06:18:04,460 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,460 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,460 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for b60b1ecdd7b636a2e033c7b1d93d483f: 2023-09-29T06:18:04,461 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:04,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1695968284462"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968284462"}]},"ts":"1695968284462"} 2023-09-29T06:18:04,464 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:04,465 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:04,465 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968284465"}]},"ts":"1695968284465"} 2023-09-29T06:18:04,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:04,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=b60b1ecdd7b636a2e033c7b1d93d483f, ASSIGN}] 2023-09-29T06:18:04,473 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=b60b1ecdd7b636a2e033c7b1d93d483f, ASSIGN 2023-09-29T06:18:04,474 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=b60b1ecdd7b636a2e033c7b1d93d483f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:04,625 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=b60b1ecdd7b636a2e033c7b1d93d483f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:04,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure b60b1ecdd7b636a2e033c7b1d93d483f, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:04,786 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,786 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7254): Opening region: {ENCODED => b60b1ecdd7b636a2e033c7b1d93d483f, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:04,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:04,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7296): checking encryption for b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7299): checking classloading for b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,789 INFO [StoreOpener-b60b1ecdd7b636a2e033c7b1d93d483f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,791 INFO [StoreOpener-b60b1ecdd7b636a2e033c7b1d93d483f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b60b1ecdd7b636a2e033c7b1d93d483f columnFamilyName testFamily 2023-09-29T06:18:04,791 DEBUG [StoreOpener-b60b1ecdd7b636a2e033c7b1d93d483f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:04,792 INFO [StoreOpener-b60b1ecdd7b636a2e033c7b1d93d483f-1 {}] regionserver.HStore(324): Store=b60b1ecdd7b636a2e033c7b1d93d483f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:04,793 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,793 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,796 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1076): writing seq id for b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:18:04,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:04,800 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): Opened b60b1ecdd7b636a2e033c7b1d93d483f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64573802, jitterRate=-0.03777536749839783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:04,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(992): Region open journal for b60b1ecdd7b636a2e033c7b1d93d483f: 2023-09-29T06:18:04,802 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., pid=26, masterSystemTime=1695968284780 2023-09-29T06:18:04,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,804 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:18:04,804 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=b60b1ecdd7b636a2e033c7b1d93d483f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:04,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=26, resume processing ppid=25 2023-09-29T06:18:04,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure b60b1ecdd7b636a2e033c7b1d93d483f, server=jenkins-hbase3.apache.org,43651,1695968188964 in 179 msec 2023-09-29T06:18:04,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=25, resume processing ppid=24 2023-09-29T06:18:04,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=b60b1ecdd7b636a2e033c7b1d93d483f, ASSIGN in 337 msec 2023-09-29T06:18:04,812 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:04,813 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968284812"}]},"ts":"1695968284812"} 2023-09-29T06:18:04,815 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:04,817 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:04,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 787 msec 2023-09-29T06:18:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:18:05,037 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2023-09-29T06:18:05,037 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:05,037 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:05,041 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:05,041 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:05,041 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2023-09-29T06:18:05,043 DEBUG [hconnection-0x2f70c637-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:05,045 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43670, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:05,054 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:05,057 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:18:05,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:18:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 88 bytes 2023-09-29T06:18:06,704 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2023-09-29T06:18:07,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:18:08,061 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:18:08,062 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-09-29T06:18:08.062Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:18:08,063 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2023-09-29T06:18:08,063 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0da78920 to 127.0.0.1:62866 2023-09-29T06:18:08,063 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:08,063 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:08,063 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:08,064 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:08,064 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7a8459cd to 127.0.0.1:62866 2023-09-29T06:18:08,075 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=208 (was 208), OpenFileDescriptor=492 (was 492), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=63 (was 69), ProcessCount=167 (was 167), AvailableMemoryMB=1832 (was 1833) 2023-09-29T06:18:08,084 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=208, OpenFileDescriptor=492, MaxFileDescriptor=60000, SystemLoadAverage=63, ProcessCount=167, AvailableMemoryMB=1831 2023-09-29T06:18:08,086 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1d4564c6 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:08,093 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x1d4564c6 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a1710c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:08,095 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x110c2ef1 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:08,098 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dc6d3dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:08,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2023-09-29T06:18:08,104 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:08,104 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:08,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2023-09-29T06:18:08,105 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=27 2023-09-29T06:18:08,517 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 381724feeefef7418ce9cb890fc7b15f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:08,572 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:18:08,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2023-09-29T06:18:08,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:08,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2023-09-29T06:18:08,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:08,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:08,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 381724feeefef7418ce9cb890fc7b15f, disabling compactions & flushes 2023-09-29T06:18:08,928 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:08,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:08,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. after waiting 0 ms 2023-09-29T06:18:08,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:08,928 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:08,928 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 381724feeefef7418ce9cb890fc7b15f: 2023-09-29T06:18:08,929 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:08,930 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1695968288929"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968288929"}]},"ts":"1695968288929"} 2023-09-29T06:18:08,932 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:08,933 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:08,933 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968288933"}]},"ts":"1695968288933"} 2023-09-29T06:18:08,935 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:08,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=381724feeefef7418ce9cb890fc7b15f, ASSIGN}] 2023-09-29T06:18:08,940 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=381724feeefef7418ce9cb890fc7b15f, ASSIGN 2023-09-29T06:18:08,941 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=381724feeefef7418ce9cb890fc7b15f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:09,092 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=381724feeefef7418ce9cb890fc7b15f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:09,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 381724feeefef7418ce9cb890fc7b15f, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=27 2023-09-29T06:18:09,252 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:09,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 381724feeefef7418ce9cb890fc7b15f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:09,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:09,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,255 INFO [StoreOpener-381724feeefef7418ce9cb890fc7b15f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,257 INFO [StoreOpener-381724feeefef7418ce9cb890fc7b15f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 381724feeefef7418ce9cb890fc7b15f columnFamilyName testFamily 2023-09-29T06:18:09,257 DEBUG [StoreOpener-381724feeefef7418ce9cb890fc7b15f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:09,258 INFO [StoreOpener-381724feeefef7418ce9cb890fc7b15f-1 {}] regionserver.HStore(324): Store=381724feeefef7418ce9cb890fc7b15f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:09,259 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,259 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:18:09,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:09,266 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 381724feeefef7418ce9cb890fc7b15f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63964742, jitterRate=-0.04685106873512268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:09,266 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 381724feeefef7418ce9cb890fc7b15f: 2023-09-29T06:18:09,267 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f., pid=29, masterSystemTime=1695968289247 2023-09-29T06:18:09,270 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:09,270 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:18:09,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=381724feeefef7418ce9cb890fc7b15f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:09,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=28 2023-09-29T06:18:09,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 381724feeefef7418ce9cb890fc7b15f, server=jenkins-hbase3.apache.org,43651,1695968188964 in 177 msec 2023-09-29T06:18:09,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-09-29T06:18:09,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=381724feeefef7418ce9cb890fc7b15f, ASSIGN in 335 msec 2023-09-29T06:18:09,277 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:09,277 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968289277"}]},"ts":"1695968289277"} 2023-09-29T06:18:09,279 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:09,283 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:09,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.1820 sec 2023-09-29T06:18:10,288 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=27 2023-09-29T06:18:11,109 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2023-09-29T06:18:11,110 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:11,110 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:11,114 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:11,114 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:11,114 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2023-09-29T06:18:11,116 DEBUG [hconnection-0x53ac9b8e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:11,119 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43348, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:11,128 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:11,131 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:11,131 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:18:11,132 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x1d4564c6 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:18:11,133 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x1d4564c6 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:11,135 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:18:11,139 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:18:11,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033012 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:18:11,140 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:18:11,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033012 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:18:11,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2023-09-29T06:18:12,344 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-29T06:18:12,344 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-29T06:18:12,705 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2023-09-29T06:18:14,142 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3001 2023-09-29T06:18:14,142 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x110c2ef1 to 127.0.0.1:62866 2023-09-29T06:18:14,142 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:14,142 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033012 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:18:14,142 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:14,142 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:14,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101033012' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:18:14,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.31.12.81:43360 deadline: 1695968295142, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101033012' does not exist 2023-09-29T06:18:14,143 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:14,143 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1d4564c6 to 127.0.0.1:62866 2023-09-29T06:18:14,143 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,43651,1695968188964 for closing scanner id = 4693677206101033012 for 381724feeefef7418ce9cb890fc7b15f of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-09-29T06:18:14,155 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=209 (was 208) - Thread LEAK? -, OpenFileDescriptor=490 (was 492), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=57 (was 63), ProcessCount=166 (was 167), AvailableMemoryMB=1839 (was 1831) - AvailableMemoryMB LEAK? - 2023-09-29T06:18:14,165 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=209, OpenFileDescriptor=490, MaxFileDescriptor=60000, SystemLoadAverage=57, ProcessCount=166, AvailableMemoryMB=1838 2023-09-29T06:18:14,167 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x50b86470 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:14,171 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x50b86470 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4907593a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:14,173 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x45cef328 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:14,180 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179bbb1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:14,182 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:14,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2023-09-29T06:18:14,185 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:14,185 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:14,186 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2023-09-29T06:18:14,186 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:18:14,198 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 320342231d3957a0d6191f34e62ec4a2, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:14,608 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:14,608 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 320342231d3957a0d6191f34e62ec4a2, disabling compactions & flushes 2023-09-29T06:18:14,608 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,608 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,608 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. after waiting 0 ms 2023-09-29T06:18:14,608 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,608 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,608 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 320342231d3957a0d6191f34e62ec4a2: 2023-09-29T06:18:14,610 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:14,610 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1695968294610"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968294610"}]},"ts":"1695968294610"} 2023-09-29T06:18:14,612 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:14,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:14,614 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968294614"}]},"ts":"1695968294614"} 2023-09-29T06:18:14,615 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:14,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=320342231d3957a0d6191f34e62ec4a2, ASSIGN}] 2023-09-29T06:18:14,621 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=320342231d3957a0d6191f34e62ec4a2, ASSIGN 2023-09-29T06:18:14,622 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=320342231d3957a0d6191f34e62ec4a2, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:14,641 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:18:14,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=320342231d3957a0d6191f34e62ec4a2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:14,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 320342231d3957a0d6191f34e62ec4a2, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:14,933 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7254): Opening region: {ENCODED => 320342231d3957a0d6191f34e62ec4a2, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:14,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:14,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7296): checking encryption for 320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7299): checking classloading for 320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,935 INFO [StoreOpener-320342231d3957a0d6191f34e62ec4a2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,937 INFO [StoreOpener-320342231d3957a0d6191f34e62ec4a2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 320342231d3957a0d6191f34e62ec4a2 columnFamilyName testFamily 2023-09-29T06:18:14,937 DEBUG [StoreOpener-320342231d3957a0d6191f34e62ec4a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:14,938 INFO [StoreOpener-320342231d3957a0d6191f34e62ec4a2-1 {}] regionserver.HStore(324): Store=320342231d3957a0d6191f34e62ec4a2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:14,939 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,942 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1076): writing seq id for 320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:18:14,945 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:14,946 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1093): Opened 320342231d3957a0d6191f34e62ec4a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69348265, jitterRate=0.033369675278663635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:14,946 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(992): Region open journal for 320342231d3957a0d6191f34e62ec4a2: 2023-09-29T06:18:14,947 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2., pid=32, masterSystemTime=1695968294928 2023-09-29T06:18:14,950 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,950 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:18:14,950 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=320342231d3957a0d6191f34e62ec4a2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:14,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=32, resume processing ppid=31 2023-09-29T06:18:14,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 320342231d3957a0d6191f34e62ec4a2, server=jenkins-hbase3.apache.org,43651,1695968188964 in 177 msec 2023-09-29T06:18:14,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-29T06:18:14,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=320342231d3957a0d6191f34e62ec4a2, ASSIGN in 335 msec 2023-09-29T06:18:14,957 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:14,957 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968294957"}]},"ts":"1695968294957"} 2023-09-29T06:18:14,959 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:14,962 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:14,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 780 msec 2023-09-29T06:18:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:18:15,189 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2023-09-29T06:18:15,190 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:15,190 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:15,194 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:15,194 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:15,194 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2023-09-29T06:18:15,196 DEBUG [hconnection-0x491df816-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:15,198 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43374, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:15,208 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:15,211 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:15,212 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033016 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:18:15,213 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2023-09-29T06:18:18,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2023-09-29T06:18:18,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:18,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2023-09-29T06:18:18,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:18,706 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2023-09-29T06:18:20,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:21,714 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2023-09-29T06:18:21,714 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x45cef328 to 127.0.0.1:62866 2023-09-29T06:18:21,714 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:21,714 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:21,715 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:21,715 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:21,715 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x50b86470 to 127.0.0.1:62866 2023-09-29T06:18:21,728 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=210 (was 209) - Thread LEAK? -, OpenFileDescriptor=487 (was 490), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=54 (was 57), ProcessCount=166 (was 166), AvailableMemoryMB=1849 (was 1838) - AvailableMemoryMB LEAK? - 2023-09-29T06:18:21,738 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=210, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=54, ProcessCount=166, AvailableMemoryMB=1849 2023-09-29T06:18:21,740 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2810785e to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:21,745 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x2810785e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292406cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:21,746 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7da74dfa to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:21,749 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453dfef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:21,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2023-09-29T06:18:21,754 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:21,754 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:21,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2023-09-29T06:18:21,755 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-09-29T06:18:22,167 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9d55fd58d40728f94dca337977664f2c, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:22,177 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:22,178 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 9d55fd58d40728f94dca337977664f2c, disabling compactions & flushes 2023-09-29T06:18:22,178 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,178 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,178 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. after waiting 0 ms 2023-09-29T06:18:22,178 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,178 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,178 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9d55fd58d40728f94dca337977664f2c: 2023-09-29T06:18:22,179 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:22,179 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1695968302179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968302179"}]},"ts":"1695968302179"} 2023-09-29T06:18:22,181 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:22,182 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:22,183 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968302183"}]},"ts":"1695968302183"} 2023-09-29T06:18:22,184 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:22,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=9d55fd58d40728f94dca337977664f2c, ASSIGN}] 2023-09-29T06:18:22,191 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=9d55fd58d40728f94dca337977664f2c, ASSIGN 2023-09-29T06:18:22,192 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=9d55fd58d40728f94dca337977664f2c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:22,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=9d55fd58d40728f94dca337977664f2c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:22,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 9d55fd58d40728f94dca337977664f2c, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:22,502 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,502 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7254): Opening region: {ENCODED => 9d55fd58d40728f94dca337977664f2c, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:22,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:22,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7296): checking encryption for 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7299): checking classloading for 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,505 INFO [StoreOpener-9d55fd58d40728f94dca337977664f2c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,506 INFO [StoreOpener-9d55fd58d40728f94dca337977664f2c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d55fd58d40728f94dca337977664f2c columnFamilyName testFamily 2023-09-29T06:18:22,506 DEBUG [StoreOpener-9d55fd58d40728f94dca337977664f2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:22,507 INFO [StoreOpener-9d55fd58d40728f94dca337977664f2c-1 {}] regionserver.HStore(324): Store=9d55fd58d40728f94dca337977664f2c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:22,508 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,508 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,511 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1076): writing seq id for 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:18:22,515 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:22,515 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): Opened 9d55fd58d40728f94dca337977664f2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66894236, jitterRate=-0.0031982064247131348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:22,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(992): Region open journal for 9d55fd58d40728f94dca337977664f2c: 2023-09-29T06:18:22,517 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c., pid=35, masterSystemTime=1695968302497 2023-09-29T06:18:22,519 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,519 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:18:22,520 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=9d55fd58d40728f94dca337977664f2c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:22,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-29T06:18:22,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 9d55fd58d40728f94dca337977664f2c, server=jenkins-hbase3.apache.org,43651,1695968188964 in 177 msec 2023-09-29T06:18:22,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=34, resume processing ppid=33 2023-09-29T06:18:22,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=9d55fd58d40728f94dca337977664f2c, ASSIGN in 335 msec 2023-09-29T06:18:22,527 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:22,527 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968302527"}]},"ts":"1695968302527"} 2023-09-29T06:18:22,529 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:22,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:22,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 780 msec 2023-09-29T06:18:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-09-29T06:18:22,758 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2023-09-29T06:18:22,758 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:22,759 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:22,763 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:22,763 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:22,763 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2023-09-29T06:18:22,765 DEBUG [hconnection-0x3e11225b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:22,768 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36710, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:22,778 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:22,781 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:22,783 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033020 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:18:22,783 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2023-09-29T06:18:24,284 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1503 2023-09-29T06:18:24,284 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7da74dfa to 127.0.0.1:62866 2023-09-29T06:18:24,284 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:24,284 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:24,285 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:24,285 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:24,285 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2810785e to 127.0.0.1:62866 2023-09-29T06:18:24,298 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=213 (was 210) - Thread LEAK? -, OpenFileDescriptor=489 (was 487) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=49 (was 54), ProcessCount=166 (was 166), AvailableMemoryMB=1848 (was 1849) 2023-09-29T06:18:24,307 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=213, OpenFileDescriptor=489, MaxFileDescriptor=60000, SystemLoadAverage=49, ProcessCount=166, AvailableMemoryMB=1848 2023-09-29T06:18:24,309 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41c5ac35 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:24,313 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x41c5ac35 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5471b0e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:24,314 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x28e8f69e to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:24,317 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edb1d30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:24,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2023-09-29T06:18:24,322 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:24,322 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:24,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2023-09-29T06:18:24,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:18:24,707 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2023-09-29T06:18:24,741 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 94654e3c565dd5486a5739c3568d8845, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:25,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:25,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 94654e3c565dd5486a5739c3568d8845, disabling compactions & flushes 2023-09-29T06:18:25,152 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. after waiting 0 ms 2023-09-29T06:18:25,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,152 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 94654e3c565dd5486a5739c3568d8845: 2023-09-29T06:18:25,154 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:25,154 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1695968305154"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968305154"}]},"ts":"1695968305154"} 2023-09-29T06:18:25,156 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:25,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:25,157 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968305157"}]},"ts":"1695968305157"} 2023-09-29T06:18:25,159 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:25,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=94654e3c565dd5486a5739c3568d8845, ASSIGN}] 2023-09-29T06:18:25,165 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=94654e3c565dd5486a5739c3568d8845, ASSIGN 2023-09-29T06:18:25,165 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=94654e3c565dd5486a5739c3568d8845, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:25,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=94654e3c565dd5486a5739c3568d8845, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:25,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 94654e3c565dd5486a5739c3568d8845, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:18:25,476 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => 94654e3c565dd5486a5739c3568d8845, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:25,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:25,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for 94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for 94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,478 INFO [StoreOpener-94654e3c565dd5486a5739c3568d8845-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,480 INFO [StoreOpener-94654e3c565dd5486a5739c3568d8845-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94654e3c565dd5486a5739c3568d8845 columnFamilyName testFamily 2023-09-29T06:18:25,480 DEBUG [StoreOpener-94654e3c565dd5486a5739c3568d8845-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:25,480 INFO [StoreOpener-94654e3c565dd5486a5739c3568d8845-1 {}] regionserver.HStore(324): Store=94654e3c565dd5486a5739c3568d8845/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:25,481 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,485 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for 94654e3c565dd5486a5739c3568d8845 2023-09-29T06:18:25,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:25,488 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened 94654e3c565dd5486a5739c3568d8845; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68974436, jitterRate=0.02779918909072876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:25,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for 94654e3c565dd5486a5739c3568d8845: 2023-09-29T06:18:25,490 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845., pid=38, masterSystemTime=1695968305471 2023-09-29T06:18:25,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,492 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:18:25,492 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=94654e3c565dd5486a5739c3568d8845, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:25,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-09-29T06:18:25,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 94654e3c565dd5486a5739c3568d8845, server=jenkins-hbase3.apache.org,43651,1695968188964 in 176 msec 2023-09-29T06:18:25,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-09-29T06:18:25,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=94654e3c565dd5486a5739c3568d8845, ASSIGN in 334 msec 2023-09-29T06:18:25,500 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:25,500 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968305500"}]},"ts":"1695968305500"} 2023-09-29T06:18:25,502 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:25,505 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:25,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 1.1860 sec 2023-09-29T06:18:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:18:27,328 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2023-09-29T06:18:27,328 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:27,328 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:27,333 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:27,333 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:27,333 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2023-09-29T06:18:27,335 DEBUG [hconnection-0x1ebfd083-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:27,337 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36712, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:27,346 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:27,350 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x413d93a0 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:27,354 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd94721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:27,354 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:27,356 DEBUG [hconnection-0x58c7dc88-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:27,361 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36726, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:27,365 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:18:27,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033025 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:18:27,366 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:18:27,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033025 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:18:27,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2023-09-29T06:18:28,005 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:28,371 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-09-29T06:18:28.371Z, RpcRetryingCaller{globalStartTime=2023-09-29T06:18:27.366Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-09-29T06:18:28.371Z, RpcRetryingCaller{globalStartTime=2023-09-29T06:18:27.366Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:18:28,372 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1005 2023-09-29T06:18:28,372 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x413d93a0 to 127.0.0.1:62866 2023-09-29T06:18:28,372 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:28,373 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x28e8f69e to 127.0.0.1:62866 2023-09-29T06:18:28,373 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:28,373 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:28,373 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:28,373 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:28,373 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41c5ac35 to 127.0.0.1:62866 2023-09-29T06:18:28,385 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=218 (was 213) - Thread LEAK? -, OpenFileDescriptor=496 (was 489) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=45 (was 49), ProcessCount=167 (was 166) - ProcessCount LEAK? -, AvailableMemoryMB=1802 (was 1848) 2023-09-29T06:18:28,396 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=218, OpenFileDescriptor=496, MaxFileDescriptor=60000, SystemLoadAverage=45, ProcessCount=167, AvailableMemoryMB=1802 2023-09-29T06:18:28,397 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x17b1c261 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:28,403 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x17b1c261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678fe547, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:28,404 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3850e210 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:28,408 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b2496a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:28,411 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:28,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2023-09-29T06:18:28,414 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:28,414 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:28,414 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2023-09-29T06:18:28,415 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-29T06:18:28,427 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c5d82055d13f28d8394959085f7e9b9c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:28,438 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:28,438 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing c5d82055d13f28d8394959085f7e9b9c, disabling compactions & flushes 2023-09-29T06:18:28,439 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,439 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,439 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. after waiting 0 ms 2023-09-29T06:18:28,439 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,439 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,439 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for c5d82055d13f28d8394959085f7e9b9c: 2023-09-29T06:18:28,440 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:28,440 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1695968308440"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968308440"}]},"ts":"1695968308440"} 2023-09-29T06:18:28,442 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:28,443 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:28,443 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968308443"}]},"ts":"1695968308443"} 2023-09-29T06:18:28,445 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:28,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=c5d82055d13f28d8394959085f7e9b9c, ASSIGN}] 2023-09-29T06:18:28,450 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=c5d82055d13f28d8394959085f7e9b9c, ASSIGN 2023-09-29T06:18:28,451 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=c5d82055d13f28d8394959085f7e9b9c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:28,602 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=c5d82055d13f28d8394959085f7e9b9c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:28,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure c5d82055d13f28d8394959085f7e9b9c, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:28,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2023-09-29T06:18:28,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:28,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2023-09-29T06:18:28,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:28,761 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,761 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => c5d82055d13f28d8394959085f7e9b9c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:28,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:28,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,764 INFO [StoreOpener-c5d82055d13f28d8394959085f7e9b9c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,765 INFO [StoreOpener-c5d82055d13f28d8394959085f7e9b9c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5d82055d13f28d8394959085f7e9b9c columnFamilyName testFamily 2023-09-29T06:18:28,765 DEBUG [StoreOpener-c5d82055d13f28d8394959085f7e9b9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:28,766 INFO [StoreOpener-c5d82055d13f28d8394959085f7e9b9c-1 {}] regionserver.HStore(324): Store=c5d82055d13f28d8394959085f7e9b9c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:28,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,770 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:18:28,772 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:28,773 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1093): Opened c5d82055d13f28d8394959085f7e9b9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72710688, jitterRate=0.08347368240356445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:28,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(992): Region open journal for c5d82055d13f28d8394959085f7e9b9c: 2023-09-29T06:18:28,775 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c., pid=41, masterSystemTime=1695968308756 2023-09-29T06:18:28,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,777 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:18:28,777 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=c5d82055d13f28d8394959085f7e9b9c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:28,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-09-29T06:18:28,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure c5d82055d13f28d8394959085f7e9b9c, server=jenkins-hbase3.apache.org,43651,1695968188964 in 175 msec 2023-09-29T06:18:28,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-09-29T06:18:28,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=c5d82055d13f28d8394959085f7e9b9c, ASSIGN in 332 msec 2023-09-29T06:18:28,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:28,784 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968308784"}]},"ts":"1695968308784"} 2023-09-29T06:18:28,785 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:28,789 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:28,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 378 msec 2023-09-29T06:18:28,867 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:18:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-29T06:18:29,418 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2023-09-29T06:18:29,418 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:29,418 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:29,422 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:29,422 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:29,422 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2023-09-29T06:18:29,424 DEBUG [hconnection-0x2db257d7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:29,426 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36742, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:29,437 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:29,439 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:29,442 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:18:29,442 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2023-09-29T06:18:30,708 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2023-09-29T06:18:30,709 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2023-09-29T06:18:32,390 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2023-09-29T06:18:32,396 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 144c806451470faf2697726823a29c67 changed from -1.0 to 0.0, refreshing cache 2023-09-29T06:18:34,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 11 bytes 2023-09-29T06:18:36,458 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:18:36,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:36742 deadline: 1695968318441, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:18:36,460 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:18:36,461 DEBUG [Listener at localhost/43419 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033029 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:18:36,461 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7022 2023-09-29T06:18:36,462 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3850e210 to 127.0.0.1:62866 2023-09-29T06:18:36,462 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:36,462 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:36,462 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:36,462 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:36,462 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x17b1c261 to 127.0.0.1:62866 2023-09-29T06:18:36,473 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=213 (was 218), OpenFileDescriptor=485 (was 496), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=50 (was 45) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=1777 (was 1802) 2023-09-29T06:18:36,483 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=213, OpenFileDescriptor=485, MaxFileDescriptor=60000, SystemLoadAverage=50, ProcessCount=167, AvailableMemoryMB=1777 2023-09-29T06:18:36,485 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x15105b39 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:36,489 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x15105b39 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a9161f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:36,490 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6778357a to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:36,493 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b34cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:36,495 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:36,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2023-09-29T06:18:36,498 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:36,498 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:36,498 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2023-09-29T06:18:36,498 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-29T06:18:36,909 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9a7d56d0a85a7d831c743aa9169ba277, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:37,319 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:37,320 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 9a7d56d0a85a7d831c743aa9169ba277, disabling compactions & flushes 2023-09-29T06:18:37,320 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,320 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,320 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. after waiting 0 ms 2023-09-29T06:18:37,320 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,320 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,320 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9a7d56d0a85a7d831c743aa9169ba277: 2023-09-29T06:18:37,321 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:37,321 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1695968317321"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968317321"}]},"ts":"1695968317321"} 2023-09-29T06:18:37,324 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:37,324 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:37,325 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968317325"}]},"ts":"1695968317325"} 2023-09-29T06:18:37,326 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:37,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=9a7d56d0a85a7d831c743aa9169ba277, ASSIGN}] 2023-09-29T06:18:37,333 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=9a7d56d0a85a7d831c743aa9169ba277, ASSIGN 2023-09-29T06:18:37,334 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=9a7d56d0a85a7d831c743aa9169ba277, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:37,485 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=9a7d56d0a85a7d831c743aa9169ba277, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:37,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 9a7d56d0a85a7d831c743aa9169ba277, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-29T06:18:37,644 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7254): Opening region: {ENCODED => 9a7d56d0a85a7d831c743aa9169ba277, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:37,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:37,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7296): checking encryption for 9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7299): checking classloading for 9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,646 INFO [StoreOpener-9a7d56d0a85a7d831c743aa9169ba277-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,648 INFO [StoreOpener-9a7d56d0a85a7d831c743aa9169ba277-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a7d56d0a85a7d831c743aa9169ba277 columnFamilyName testFamily 2023-09-29T06:18:37,648 DEBUG [StoreOpener-9a7d56d0a85a7d831c743aa9169ba277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:37,648 INFO [StoreOpener-9a7d56d0a85a7d831c743aa9169ba277-1 {}] regionserver.HStore(324): Store=9a7d56d0a85a7d831c743aa9169ba277/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:37,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,654 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1076): writing seq id for 9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:18:37,656 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:37,657 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1093): Opened 9a7d56d0a85a7d831c743aa9169ba277; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63807770, jitterRate=-0.049190133810043335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:37,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(992): Region open journal for 9a7d56d0a85a7d831c743aa9169ba277: 2023-09-29T06:18:37,659 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., pid=44, masterSystemTime=1695968317639 2023-09-29T06:18:37,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,661 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:18:37,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=9a7d56d0a85a7d831c743aa9169ba277, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:37,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-09-29T06:18:37,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 9a7d56d0a85a7d831c743aa9169ba277, server=jenkins-hbase3.apache.org,43651,1695968188964 in 176 msec 2023-09-29T06:18:37,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-09-29T06:18:37,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=9a7d56d0a85a7d831c743aa9169ba277, ASSIGN in 335 msec 2023-09-29T06:18:37,668 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:37,669 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968317668"}]},"ts":"1695968317668"} 2023-09-29T06:18:37,670 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:37,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:37,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.1780 sec 2023-09-29T06:18:38,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2023-09-29T06:18:38,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:38,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2023-09-29T06:18:38,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-29T06:18:39,503 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2023-09-29T06:18:39,503 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:39,503 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:39,507 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:39,507 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:39,507 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2023-09-29T06:18:39,509 DEBUG [hconnection-0x194540d0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:39,512 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55848, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:39,520 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:39,523 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:18:39,523 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:18:39,524 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x15105b39 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:18:39,524 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x15105b39 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:39,526 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55856, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:18:39,529 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:18:41,217 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101032980 expired clientIPAndPort=172.31.12.81:51600, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:18:43,146 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:43,194 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2023-09-29T06:18:45,532 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2023-09-29T06:18:45,532 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2023-09-29T06:18:45,532 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:18:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 88 bytes 2023-09-29T06:18:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:18:46,547 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:18:48,223 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101032982 expired clientIPAndPort=172.31.12.81:51600, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:18:48,530 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms 2023-09-29T06:18:48,531 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms 2023-09-29T06:18:48,531 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:18:48,532 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-09-29T06:18:45.533Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2023-09-29T06:18:48.531Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-09-29T06:18:48,534 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9011 2023-09-29T06:18:48,534 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6778357a to 127.0.0.1:62866 2023-09-29T06:18:48,534 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:48,534 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:48,534 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:48,534 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:48,534 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x15105b39 to 127.0.0.1:62866 2023-09-29T06:18:48,546 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=211 (was 213), OpenFileDescriptor=478 (was 485), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=47 (was 50), ProcessCount=168 (was 167) - ProcessCount LEAK? -, AvailableMemoryMB=1741 (was 1777) 2023-09-29T06:18:48,556 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=211, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=47, ProcessCount=168, AvailableMemoryMB=1741 2023-09-29T06:18:48,557 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0348e83e to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:48,561 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x0348e83e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2402c82d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:48,562 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x07a3281f to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:48,568 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c51d87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:48,571 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:48,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2023-09-29T06:18:48,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:48,574 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:48,575 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2023-09-29T06:18:48,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-29T06:18:48,597 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b7d72ea3cc8715208649befa0c48f46e, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:49,011 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:49,011 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing b7d72ea3cc8715208649befa0c48f46e, disabling compactions & flushes 2023-09-29T06:18:49,011 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,011 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,011 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. after waiting 0 ms 2023-09-29T06:18:49,011 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,011 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,011 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for b7d72ea3cc8715208649befa0c48f46e: 2023-09-29T06:18:49,013 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:49,013 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1695968329013"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968329013"}]},"ts":"1695968329013"} 2023-09-29T06:18:49,015 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:49,016 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:49,016 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968329016"}]},"ts":"1695968329016"} 2023-09-29T06:18:49,018 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:49,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=b7d72ea3cc8715208649befa0c48f46e, ASSIGN}] 2023-09-29T06:18:49,024 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=b7d72ea3cc8715208649befa0c48f46e, ASSIGN 2023-09-29T06:18:49,025 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=b7d72ea3cc8715208649befa0c48f46e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:49,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=b7d72ea3cc8715208649befa0c48f46e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:49,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure b7d72ea3cc8715208649befa0c48f46e, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:49,338 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7254): Opening region: {ENCODED => b7d72ea3cc8715208649befa0c48f46e, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:49,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:49,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7296): checking encryption for b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7299): checking classloading for b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,341 INFO [StoreOpener-b7d72ea3cc8715208649befa0c48f46e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,343 INFO [StoreOpener-b7d72ea3cc8715208649befa0c48f46e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d72ea3cc8715208649befa0c48f46e columnFamilyName testFamily 2023-09-29T06:18:49,343 DEBUG [StoreOpener-b7d72ea3cc8715208649befa0c48f46e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:49,343 INFO [StoreOpener-b7d72ea3cc8715208649befa0c48f46e-1 {}] regionserver.HStore(324): Store=b7d72ea3cc8715208649befa0c48f46e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:49,344 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,345 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1076): writing seq id for b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:18:49,351 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:49,352 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1093): Opened b7d72ea3cc8715208649befa0c48f46e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61869979, jitterRate=-0.07806546986103058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:49,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(992): Region open journal for b7d72ea3cc8715208649befa0c48f46e: 2023-09-29T06:18:49,354 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e., pid=47, masterSystemTime=1695968329330 2023-09-29T06:18:49,356 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,356 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:18:49,357 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=b7d72ea3cc8715208649befa0c48f46e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:49,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-09-29T06:18:49,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure b7d72ea3cc8715208649befa0c48f46e, server=jenkins-hbase3.apache.org,43651,1695968188964 in 181 msec 2023-09-29T06:18:49,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-09-29T06:18:49,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=b7d72ea3cc8715208649befa0c48f46e, ASSIGN in 338 msec 2023-09-29T06:18:49,364 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:49,364 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968329364"}]},"ts":"1695968329364"} 2023-09-29T06:18:49,366 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:49,369 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:49,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 798 msec 2023-09-29T06:18:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-29T06:18:49,579 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2023-09-29T06:18:49,579 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:49,579 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:49,582 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:49,583 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:49,583 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2023-09-29T06:18:49,585 DEBUG [hconnection-0x3ef57a3a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:49,586 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60210, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:49,595 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:49,600 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5a51fc25 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:49,604 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16720967, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:49,604 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:18:49,606 DEBUG [hconnection-0x6c544b9a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:49,608 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60226, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:49,612 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:18:49,613 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033042 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:18:49,613 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:18:49,614 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033042 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:18:49,614 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2023-09-29T06:18:53,047 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:18:54,891 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:18:55,196 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2023-09-29T06:18:55,622 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-09-29T06:18:55.621Z, RpcRetryingCaller{globalStartTime=2023-09-29T06:18:49.614Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-09-29T06:18:55.621Z, RpcRetryingCaller{globalStartTime=2023-09-29T06:18:49.614Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:18:55,623 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6008 2023-09-29T06:18:55,623 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5a51fc25 to 127.0.0.1:62866 2023-09-29T06:18:55,623 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:55,623 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x07a3281f to 127.0.0.1:62866 2023-09-29T06:18:55,623 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:55,623 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:18:55,623 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:18:55,624 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:18:55,624 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0348e83e to 127.0.0.1:62866 2023-09-29T06:18:55,635 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=216 (was 211) - Thread LEAK? -, OpenFileDescriptor=486 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=59 (was 47) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 168), AvailableMemoryMB=1711 (was 1741) 2023-09-29T06:18:55,646 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=216, OpenFileDescriptor=486, MaxFileDescriptor=60000, SystemLoadAverage=59, ProcessCount=168, AvailableMemoryMB=1711 2023-09-29T06:18:55,648 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x211f90ca to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:55,654 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x211f90ca {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5a0686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:55,655 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5df12ff8 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:18:55,658 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24514629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:18:55,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:18:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2023-09-29T06:18:55,663 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:18:55,663 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:55,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2023-09-29T06:18:55,664 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:18:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:18:56,080 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c6e7c51cc3296370de4e4c0324ab2c28, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:18:56,115 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:18:56,330 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101032987 expired clientIPAndPort=172.31.12.81:46574, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:18:56,491 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:56,491 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing c6e7c51cc3296370de4e4c0324ab2c28, disabling compactions & flushes 2023-09-29T06:18:56,491 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,491 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,491 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. after waiting 0 ms 2023-09-29T06:18:56,491 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,491 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,491 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for c6e7c51cc3296370de4e4c0324ab2c28: 2023-09-29T06:18:56,493 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:18:56,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1695968336493"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968336493"}]},"ts":"1695968336493"} 2023-09-29T06:18:56,496 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:18:56,497 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:18:56,497 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968336497"}]},"ts":"1695968336497"} 2023-09-29T06:18:56,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2023-09-29T06:18:56,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c6e7c51cc3296370de4e4c0324ab2c28, ASSIGN}] 2023-09-29T06:18:56,506 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c6e7c51cc3296370de4e4c0324ab2c28, ASSIGN 2023-09-29T06:18:56,507 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c6e7c51cc3296370de4e4c0324ab2c28, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:18:56,658 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=c6e7c51cc3296370de4e4c0324ab2c28, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:56,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure c6e7c51cc3296370de4e4c0324ab2c28, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:18:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:18:56,818 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,818 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7254): Opening region: {ENCODED => c6e7c51cc3296370de4e4c0324ab2c28, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:18:56,818 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,818 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:18:56,818 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7296): checking encryption for c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,818 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7299): checking classloading for c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,820 INFO [StoreOpener-c6e7c51cc3296370de4e4c0324ab2c28-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,822 INFO [StoreOpener-c6e7c51cc3296370de4e4c0324ab2c28-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6e7c51cc3296370de4e4c0324ab2c28 columnFamilyName testFamily 2023-09-29T06:18:56,822 DEBUG [StoreOpener-c6e7c51cc3296370de4e4c0324ab2c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:18:56,822 INFO [StoreOpener-c6e7c51cc3296370de4e4c0324ab2c28-1 {}] regionserver.HStore(324): Store=c6e7c51cc3296370de4e4c0324ab2c28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:18:56,823 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,824 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1076): writing seq id for c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:18:56,828 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:18:56,829 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): Opened c6e7c51cc3296370de4e4c0324ab2c28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69365582, jitterRate=0.033627718687057495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:18:56,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(992): Region open journal for c6e7c51cc3296370de4e4c0324ab2c28: 2023-09-29T06:18:56,831 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., pid=50, masterSystemTime=1695968336813 2023-09-29T06:18:56,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,833 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:18:56,833 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=c6e7c51cc3296370de4e4c0324ab2c28, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:18:56,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=50, resume processing ppid=49 2023-09-29T06:18:56,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure c6e7c51cc3296370de4e4c0324ab2c28, server=jenkins-hbase3.apache.org,43651,1695968188964 in 175 msec 2023-09-29T06:18:56,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-09-29T06:18:56,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c6e7c51cc3296370de4e4c0324ab2c28, ASSIGN in 333 msec 2023-09-29T06:18:56,840 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:18:56,840 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968336840"}]},"ts":"1695968336840"} 2023-09-29T06:18:56,841 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2023-09-29T06:18:56,844 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:18:56,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1830 sec 2023-09-29T06:18:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:18:58,669 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2023-09-29T06:18:58,669 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2023-09-29T06:18:58,669 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:58,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2023-09-29T06:18:58,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:58,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2023-09-29T06:18:58,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2023-09-29T06:18:58,673 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2023-09-29T06:18:58,673 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:18:58,673 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2023-09-29T06:18:58,675 DEBUG [hconnection-0x58a7acb2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:58,677 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:58,685 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:18:58,688 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:18:58,689 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:18:58,690 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x211f90ca {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:18:58,690 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x211f90ca {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:18:58,691 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60540, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:18:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:18:58,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:18:58,932 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101032994 expired clientIPAndPort=172.31.12.81:58046, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:18:58,932 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101032993 expired clientIPAndPort=172.31.12.81:58046, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:18:58,932 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101032991 expired clientIPAndPort=172.31.12.81:58046, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:18:59,700 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2023-09-29T06:18:59,701 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2023-09-29T06:18:59,701 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:19:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 88 bytes 2023-09-29T06:19:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:19:00,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:19:01,197 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2023-09-29T06:19:01,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms 2023-09-29T06:19:01,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms 2023-09-29T06:19:01,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:19:01,692 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-09-29T06:18:59.701Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2023-09-29T06:19:01.691Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-09-29T06:19:01,694 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3006 2023-09-29T06:19:01,695 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5df12ff8 to 127.0.0.1:62866 2023-09-29T06:19:01,695 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:01,695 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:01,695 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:01,695 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:01,695 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x211f90ca to 127.0.0.1:62866 2023-09-29T06:19:01,706 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=214 (was 216), OpenFileDescriptor=483 (was 486), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=54 (was 59), ProcessCount=167 (was 168), AvailableMemoryMB=1708 (was 1711) 2023-09-29T06:19:01,717 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=214, OpenFileDescriptor=483, MaxFileDescriptor=60000, SystemLoadAverage=54, ProcessCount=167, AvailableMemoryMB=1708 2023-09-29T06:19:01,718 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x57f251f1 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:01,723 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x57f251f1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b86d45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:01,724 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6c1b6394 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:01,728 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6c443e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:01,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2023-09-29T06:19:01,733 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:01,733 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:01,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2023-09-29T06:19:01,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-09-29T06:19:02,147 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => eb4991a9a0ce4f84f517a97a03fff332, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:02,217 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:19:02,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:02,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:02,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing eb4991a9a0ce4f84f517a97a03fff332, disabling compactions & flushes 2023-09-29T06:19:02,557 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. after waiting 0 ms 2023-09-29T06:19:02,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,558 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,558 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for eb4991a9a0ce4f84f517a97a03fff332: 2023-09-29T06:19:02,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:02,559 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1695968342559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968342559"}]},"ts":"1695968342559"} 2023-09-29T06:19:02,561 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:02,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:02,562 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968342562"}]},"ts":"1695968342562"} 2023-09-29T06:19:02,563 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:02,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eb4991a9a0ce4f84f517a97a03fff332, ASSIGN}] 2023-09-29T06:19:02,568 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eb4991a9a0ce4f84f517a97a03fff332, ASSIGN 2023-09-29T06:19:02,569 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eb4991a9a0ce4f84f517a97a03fff332, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:02,720 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=eb4991a9a0ce4f84f517a97a03fff332, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:02,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure eb4991a9a0ce4f84f517a97a03fff332, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-09-29T06:19:02,880 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,881 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => eb4991a9a0ce4f84f517a97a03fff332, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:02,881 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,881 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:02,881 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,881 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,883 INFO [StoreOpener-eb4991a9a0ce4f84f517a97a03fff332-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,884 INFO [StoreOpener-eb4991a9a0ce4f84f517a97a03fff332-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb4991a9a0ce4f84f517a97a03fff332 columnFamilyName testFamily 2023-09-29T06:19:02,885 DEBUG [StoreOpener-eb4991a9a0ce4f84f517a97a03fff332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:02,885 INFO [StoreOpener-eb4991a9a0ce4f84f517a97a03fff332-1 {}] regionserver.HStore(324): Store=eb4991a9a0ce4f84f517a97a03fff332/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:02,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,889 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:19:02,891 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:02,892 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened eb4991a9a0ce4f84f517a97a03fff332; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66374269, jitterRate=-0.010946318507194519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:02,915 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for eb4991a9a0ce4f84f517a97a03fff332: 2023-09-29T06:19:02,916 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332., pid=53, masterSystemTime=1695968342874 2023-09-29T06:19:02,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,918 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:19:02,919 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=eb4991a9a0ce4f84f517a97a03fff332, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:02,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-09-29T06:19:02,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure eb4991a9a0ce4f84f517a97a03fff332, server=jenkins-hbase3.apache.org,43651,1695968188964 in 199 msec 2023-09-29T06:19:02,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-09-29T06:19:02,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eb4991a9a0ce4f84f517a97a03fff332, ASSIGN in 355 msec 2023-09-29T06:19:02,925 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:02,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968342925"}]},"ts":"1695968342925"} 2023-09-29T06:19:02,927 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:02,930 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:02,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1990 sec 2023-09-29T06:19:04,036 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033000 expired clientIPAndPort=172.31.12.81:43666, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:19:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-09-29T06:19:04,738 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2023-09-29T06:19:04,738 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:04,738 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:04,742 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:04,742 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:04,742 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2023-09-29T06:19:04,744 DEBUG [hconnection-0x1a3eb693-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:04,746 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39812, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:04,754 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:04,756 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-09-29T06:19:04,756 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:19:04,757 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x57f251f1 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:19:04,758 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x57f251f1 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:04,759 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:19:04,762 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-09-29T06:19:04,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033054 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:04,763 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-09-29T06:19:04,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033054 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:04,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101033054 2023-09-29T06:19:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:39820 deadline: 1695968347762, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:04,764 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033054 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:04,765 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-09-29T06:19:04,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033055 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:04,765 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-09-29T06:19:04,766 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:04,766 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:04,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033056 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:04,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101033056 2023-09-29T06:19:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:39820 deadline: 1695968347765, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:04,767 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033056 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:04,768 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:04,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033057 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:04,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4693677206101033057 2023-09-29T06:19:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:39820 deadline: 1695968347768, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:04,769 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033057 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:04,770 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:04,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033058 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:04,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4693677206101033058 2023-09-29T06:19:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:39820 deadline: 1695968347769, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:04,772 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033058 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:04,773 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:04,773 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6c1b6394 to 127.0.0.1:62866 2023-09-29T06:19:04,773 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:04,774 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:04,774 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:04,774 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:04,774 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x57f251f1 to 127.0.0.1:62866 2023-09-29T06:19:04,785 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=216 (was 214) - Thread LEAK? -, OpenFileDescriptor=484 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=50 (was 54), ProcessCount=167 (was 167), AvailableMemoryMB=1629 (was 1708) 2023-09-29T06:19:04,796 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=216, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=50, ProcessCount=167, AvailableMemoryMB=1629 2023-09-29T06:19:04,798 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x56df85a5 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:04,802 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x56df85a5 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564383a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:04,803 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7c3ca03a to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:04,809 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c6f51c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:04,811 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:04,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2023-09-29T06:19:04,814 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:04,814 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:04,815 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:04,815 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2023-09-29T06:19:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:19:05,232 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ee80440612009a663af11a4db3cdc824, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:05,650 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:05,650 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing ee80440612009a663af11a4db3cdc824, disabling compactions & flushes 2023-09-29T06:19:05,650 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,650 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,650 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. after waiting 0 ms 2023-09-29T06:19:05,650 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,650 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,650 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for ee80440612009a663af11a4db3cdc824: 2023-09-29T06:19:05,652 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:05,652 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1695968345652"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968345652"}]},"ts":"1695968345652"} 2023-09-29T06:19:05,654 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:05,655 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:05,655 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968345655"}]},"ts":"1695968345655"} 2023-09-29T06:19:05,656 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:05,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ee80440612009a663af11a4db3cdc824, ASSIGN}] 2023-09-29T06:19:05,662 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ee80440612009a663af11a4db3cdc824, ASSIGN 2023-09-29T06:19:05,662 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ee80440612009a663af11a4db3cdc824, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:05,813 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ee80440612009a663af11a4db3cdc824, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:05,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure ee80440612009a663af11a4db3cdc824, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:19:05,972 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7254): Opening region: {ENCODED => ee80440612009a663af11a4db3cdc824, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:05,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:05,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7296): checking encryption for ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,973 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7299): checking classloading for ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,974 INFO [StoreOpener-ee80440612009a663af11a4db3cdc824-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,975 INFO [StoreOpener-ee80440612009a663af11a4db3cdc824-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee80440612009a663af11a4db3cdc824 columnFamilyName testFamily 2023-09-29T06:19:05,975 DEBUG [StoreOpener-ee80440612009a663af11a4db3cdc824-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:05,976 INFO [StoreOpener-ee80440612009a663af11a4db3cdc824-1 {}] regionserver.HStore(324): Store=ee80440612009a663af11a4db3cdc824/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:05,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,979 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1076): writing seq id for ee80440612009a663af11a4db3cdc824 2023-09-29T06:19:05,982 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:05,982 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1093): Opened ee80440612009a663af11a4db3cdc824; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66699184, jitterRate=-0.006104707717895508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:05,983 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(992): Region open journal for ee80440612009a663af11a4db3cdc824: 2023-09-29T06:19:05,984 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824., pid=56, masterSystemTime=1695968345967 2023-09-29T06:19:05,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,986 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:19:05,987 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ee80440612009a663af11a4db3cdc824, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:05,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-09-29T06:19:05,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure ee80440612009a663af11a4db3cdc824, server=jenkins-hbase3.apache.org,43651,1695968188964 in 173 msec 2023-09-29T06:19:05,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-09-29T06:19:05,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=ee80440612009a663af11a4db3cdc824, ASSIGN in 331 msec 2023-09-29T06:19:05,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:05,994 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968345994"}]},"ts":"1695968345994"} 2023-09-29T06:19:05,995 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:05,998 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:05,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1870 sec 2023-09-29T06:19:06,639 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033005 expired clientIPAndPort=172.31.12.81:43670, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:19:07,198 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2023-09-29T06:19:07,198 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2023-09-29T06:19:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:19:07,820 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2023-09-29T06:19:07,820 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:07,820 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:07,824 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:07,824 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:07,824 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2023-09-29T06:19:07,826 DEBUG [hconnection-0x5cc9987b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:07,828 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39834, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:07,835 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:07,837 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:19:07,838 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:19:08,383 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:08,640 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033007 expired clientIPAndPort=172.31.12.81:43670, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:19:08,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2023-09-29T06:19:08,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:08,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2023-09-29T06:19:08,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 88 bytes 2023-09-29T06:19:14,645 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033012 expired clientIPAndPort=172.31.12.81:43360, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:19:14,855 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:19:16,841 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:19:16,842 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-09-29T06:19:16.842Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1985ms, rpcTimeout=1982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:19:16,842 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2023-09-29T06:19:16,843 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7c3ca03a to 127.0.0.1:62866 2023-09-29T06:19:16,843 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:16,843 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:16,843 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:16,844 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:16,844 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x56df85a5 to 127.0.0.1:62866 2023-09-29T06:19:16,854 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=212 (was 216), OpenFileDescriptor=481 (was 484), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=42 (was 50), ProcessCount=167 (was 167), AvailableMemoryMB=1624 (was 1629) 2023-09-29T06:19:16,864 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=212, OpenFileDescriptor=481, MaxFileDescriptor=60000, SystemLoadAverage=42, ProcessCount=167, AvailableMemoryMB=1624 2023-09-29T06:19:16,865 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x14726221 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:16,873 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x14726221 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e999d3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:16,874 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3aac1b39 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:16,877 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78dc154e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:16,879 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:16,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2023-09-29T06:19:16,881 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:16,882 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:16,882 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2023-09-29T06:19:16,882 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-29T06:19:17,298 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 6c193cd68953fa128ebdeb920ceacb28, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:17,711 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:17,711 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 6c193cd68953fa128ebdeb920ceacb28, disabling compactions & flushes 2023-09-29T06:19:17,711 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:17,711 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:17,711 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. after waiting 0 ms 2023-09-29T06:19:17,711 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:17,711 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:17,711 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 6c193cd68953fa128ebdeb920ceacb28: 2023-09-29T06:19:17,712 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:17,713 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1695968357712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968357712"}]},"ts":"1695968357712"} 2023-09-29T06:19:17,714 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:17,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:17,715 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968357715"}]},"ts":"1695968357715"} 2023-09-29T06:19:17,717 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:17,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=6c193cd68953fa128ebdeb920ceacb28, ASSIGN}] 2023-09-29T06:19:17,722 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=6c193cd68953fa128ebdeb920ceacb28, ASSIGN 2023-09-29T06:19:17,723 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=6c193cd68953fa128ebdeb920ceacb28, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:17,874 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=6c193cd68953fa128ebdeb920ceacb28, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:17,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 6c193cd68953fa128ebdeb920ceacb28, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-29T06:19:18,034 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:18,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => 6c193cd68953fa128ebdeb920ceacb28, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:18,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:18,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for 6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for 6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,036 INFO [StoreOpener-6c193cd68953fa128ebdeb920ceacb28-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,037 INFO [StoreOpener-6c193cd68953fa128ebdeb920ceacb28-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c193cd68953fa128ebdeb920ceacb28 columnFamilyName testFamily 2023-09-29T06:19:18,038 DEBUG [StoreOpener-6c193cd68953fa128ebdeb920ceacb28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:18,038 INFO [StoreOpener-6c193cd68953fa128ebdeb920ceacb28-1 {}] regionserver.HStore(324): Store=6c193cd68953fa128ebdeb920ceacb28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:18,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for 6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:19:18,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:18,044 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1093): Opened 6c193cd68953fa128ebdeb920ceacb28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62213333, jitterRate=-0.07294909656047821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:18,045 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(992): Region open journal for 6c193cd68953fa128ebdeb920ceacb28: 2023-09-29T06:19:18,046 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28., pid=59, masterSystemTime=1695968358028 2023-09-29T06:19:18,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:18,047 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:19:18,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=6c193cd68953fa128ebdeb920ceacb28, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:18,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-09-29T06:19:18,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 6c193cd68953fa128ebdeb920ceacb28, server=jenkins-hbase3.apache.org,43651,1695968188964 in 173 msec 2023-09-29T06:19:18,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-09-29T06:19:18,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=6c193cd68953fa128ebdeb920ceacb28, ASSIGN in 331 msec 2023-09-29T06:19:18,053 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:18,053 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968358053"}]},"ts":"1695968358053"} 2023-09-29T06:19:18,054 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:18,057 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:18,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 1.1780 sec 2023-09-29T06:19:18,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2023-09-29T06:19:18,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:19,199 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2023-09-29T06:19:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-29T06:19:19,887 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2023-09-29T06:19:19,887 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:19,887 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:19,891 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:19,891 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:19,891 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2023-09-29T06:19:19,894 DEBUG [hconnection-0x355a96be-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:19,895 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47778, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:19,903 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:19,905 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:19:19,905 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:19:19,906 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x14726221 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:19:19,906 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x14726221 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:19,907 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:19:19,910 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:19:19,910 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033070 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:19,911 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:19:19,911 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033070 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:19,911 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2023-09-29T06:19:21,356 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:19:23,570 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:28,921 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9010 2023-09-29T06:19:28,921 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3aac1b39 to 127.0.0.1:62866 2023-09-29T06:19:28,921 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033070 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:28,921 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:28,921 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:28,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101033070' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:19:28,921 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:28,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.31.12.81:47780 deadline: 1695968374921, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101033070' does not exist 2023-09-29T06:19:28,922 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:28,922 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x14726221 to 127.0.0.1:62866 2023-09-29T06:19:28,922 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,43651,1695968188964 for closing scanner id = 4693677206101033070 for 6c193cd68953fa128ebdeb920ceacb28 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-09-29T06:19:28,934 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=213 (was 212) - Thread LEAK? -, OpenFileDescriptor=480 (was 481), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=40 (was 42), ProcessCount=167 (was 167), AvailableMemoryMB=1533 (was 1624) 2023-09-29T06:19:28,944 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=213, OpenFileDescriptor=480, MaxFileDescriptor=60000, SystemLoadAverage=40, ProcessCount=167, AvailableMemoryMB=1532 2023-09-29T06:19:28,945 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x663ac084 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:28,949 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x663ac084 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3b8546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:28,950 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2a0194a2 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:28,955 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6a061f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:28,956 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033025 expired clientIPAndPort=172.31.12.81:36726, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:19:28,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2023-09-29T06:19:28,959 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:28,959 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:28,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2023-09-29T06:19:28,960 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:19:29,372 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cb052ffb687b3cbe7ab95c4d907309a7, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:29,412 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:19:29,783 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:29,783 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing cb052ffb687b3cbe7ab95c4d907309a7, disabling compactions & flushes 2023-09-29T06:19:29,783 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:29,783 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:29,783 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. after waiting 0 ms 2023-09-29T06:19:29,783 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:29,783 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:29,783 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for cb052ffb687b3cbe7ab95c4d907309a7: 2023-09-29T06:19:29,784 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:29,785 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1695968369785"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968369785"}]},"ts":"1695968369785"} 2023-09-29T06:19:29,787 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:29,787 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:29,788 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968369788"}]},"ts":"1695968369788"} 2023-09-29T06:19:29,789 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:29,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=cb052ffb687b3cbe7ab95c4d907309a7, ASSIGN}] 2023-09-29T06:19:29,798 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=cb052ffb687b3cbe7ab95c4d907309a7, ASSIGN 2023-09-29T06:19:29,799 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=cb052ffb687b3cbe7ab95c4d907309a7, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:29,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=cb052ffb687b3cbe7ab95c4d907309a7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:29,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure cb052ffb687b3cbe7ab95c4d907309a7, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:19:30,111 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:30,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7254): Opening region: {ENCODED => cb052ffb687b3cbe7ab95c4d907309a7, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:30,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:30,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7296): checking encryption for cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7299): checking classloading for cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,114 INFO [StoreOpener-cb052ffb687b3cbe7ab95c4d907309a7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,115 INFO [StoreOpener-cb052ffb687b3cbe7ab95c4d907309a7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb052ffb687b3cbe7ab95c4d907309a7 columnFamilyName testFamily 2023-09-29T06:19:30,115 DEBUG [StoreOpener-cb052ffb687b3cbe7ab95c4d907309a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:30,116 INFO [StoreOpener-cb052ffb687b3cbe7ab95c4d907309a7-1 {}] regionserver.HStore(324): Store=cb052ffb687b3cbe7ab95c4d907309a7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:30,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,120 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1076): writing seq id for cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:19:30,122 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:30,123 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1093): Opened cb052ffb687b3cbe7ab95c4d907309a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65001084, jitterRate=-0.03140836954116821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:30,123 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(992): Region open journal for cb052ffb687b3cbe7ab95c4d907309a7: 2023-09-29T06:19:30,124 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7., pid=62, masterSystemTime=1695968370105 2023-09-29T06:19:30,126 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:30,126 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:19:30,126 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=cb052ffb687b3cbe7ab95c4d907309a7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:30,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=62, resume processing ppid=61 2023-09-29T06:19:30,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure cb052ffb687b3cbe7ab95c4d907309a7, server=jenkins-hbase3.apache.org,43651,1695968188964 in 176 msec 2023-09-29T06:19:30,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-09-29T06:19:30,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=cb052ffb687b3cbe7ab95c4d907309a7, ASSIGN in 334 msec 2023-09-29T06:19:30,132 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:30,132 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968370132"}]},"ts":"1695968370132"} 2023-09-29T06:19:30,133 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:30,135 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:30,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1780 sec 2023-09-29T06:19:31,201 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2023-09-29T06:19:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:19:31,964 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2023-09-29T06:19:31,964 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:31,964 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:31,968 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:31,968 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:31,968 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2023-09-29T06:19:31,970 DEBUG [hconnection-0x349ed019-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:31,972 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49836, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:31,979 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:31,981 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-09-29T06:19:31,983 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-09-29T06:19:31,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033074 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:31,984 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-09-29T06:19:31,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033074 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:31,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101033074 2023-09-29T06:19:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:49836 deadline: 1695968374984, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:31,986 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:19:31,987 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-09-29T06:19:31,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033075 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:31,989 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-09-29T06:19:31,989 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:31,990 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:31,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033076 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:31,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=4693677206101033076 2023-09-29T06:19:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:49836 deadline: 1695968374990, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:31,991 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:19:31,993 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:31,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033077 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:31,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=4693677206101033077 2023-09-29T06:19:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:49836 deadline: 1695968374993, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:31,995 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:19:31,996 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:31,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033078 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:31,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=4693677206101033078 2023-09-29T06:19:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:49836 deadline: 1695968374997, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-09-29T06:19:31,998 DEBUG [Listener at localhost/43419 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:19:31,999 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-09-29T06:19:32,000 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2a0194a2 to 127.0.0.1:62866 2023-09-29T06:19:32,000 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:32,000 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:32,000 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:32,000 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:32,000 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x663ac084 to 127.0.0.1:62866 2023-09-29T06:19:32,012 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=216 (was 213) - Thread LEAK? -, OpenFileDescriptor=484 (was 480) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=36 (was 40), ProcessCount=167 (was 167), AvailableMemoryMB=1531 (was 1532) 2023-09-29T06:19:32,022 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=216, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=36, ProcessCount=167, AvailableMemoryMB=1531 2023-09-29T06:19:32,023 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x17afbb8b to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:32,027 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x17afbb8b {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acb9312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:32,028 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6c6bdf58 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:32,031 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66567a8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:32,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2023-09-29T06:19:32,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:32,036 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:32,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2023-09-29T06:19:32,037 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=63 2023-09-29T06:19:32,448 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ffd661ca8f6efec9237d72fe94e42cb3, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:32,858 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:32,858 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing ffd661ca8f6efec9237d72fe94e42cb3, disabling compactions & flushes 2023-09-29T06:19:32,858 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:32,858 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:32,858 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. after waiting 0 ms 2023-09-29T06:19:32,858 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:32,858 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:32,858 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for ffd661ca8f6efec9237d72fe94e42cb3: 2023-09-29T06:19:32,860 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:32,860 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1695968372860"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968372860"}]},"ts":"1695968372860"} 2023-09-29T06:19:32,862 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:32,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:32,863 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968372862"}]},"ts":"1695968372862"} 2023-09-29T06:19:32,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:32,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ffd661ca8f6efec9237d72fe94e42cb3, ASSIGN}] 2023-09-29T06:19:32,870 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ffd661ca8f6efec9237d72fe94e42cb3, ASSIGN 2023-09-29T06:19:32,871 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ffd661ca8f6efec9237d72fe94e42cb3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:33,022 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=ffd661ca8f6efec9237d72fe94e42cb3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:33,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure ffd661ca8f6efec9237d72fe94e42cb3, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=63 2023-09-29T06:19:33,180 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:33,180 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7254): Opening region: {ENCODED => ffd661ca8f6efec9237d72fe94e42cb3, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:33,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:33,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7296): checking encryption for ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7299): checking classloading for ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,182 INFO [StoreOpener-ffd661ca8f6efec9237d72fe94e42cb3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,184 INFO [StoreOpener-ffd661ca8f6efec9237d72fe94e42cb3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffd661ca8f6efec9237d72fe94e42cb3 columnFamilyName testFamily 2023-09-29T06:19:33,184 DEBUG [StoreOpener-ffd661ca8f6efec9237d72fe94e42cb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:33,184 INFO [StoreOpener-ffd661ca8f6efec9237d72fe94e42cb3-1 {}] regionserver.HStore(324): Store=ffd661ca8f6efec9237d72fe94e42cb3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:33,185 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,186 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1076): writing seq id for ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:19:33,190 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:33,190 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): Opened ffd661ca8f6efec9237d72fe94e42cb3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64312471, jitterRate=-0.04166950285434723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:33,191 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(992): Region open journal for ffd661ca8f6efec9237d72fe94e42cb3: 2023-09-29T06:19:33,192 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3., pid=65, masterSystemTime=1695968373176 2023-09-29T06:19:33,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:33,193 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:19:33,194 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=ffd661ca8f6efec9237d72fe94e42cb3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:33,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-09-29T06:19:33,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure ffd661ca8f6efec9237d72fe94e42cb3, server=jenkins-hbase3.apache.org,43651,1695968188964 in 171 msec 2023-09-29T06:19:33,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=64, resume processing ppid=63 2023-09-29T06:19:33,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ffd661ca8f6efec9237d72fe94e42cb3, ASSIGN in 328 msec 2023-09-29T06:19:33,200 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:33,200 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968373200"}]},"ts":"1695968373200"} 2023-09-29T06:19:33,201 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:33,203 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:33,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.1700 sec 2023-09-29T06:19:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=63 2023-09-29T06:19:35,041 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2023-09-29T06:19:35,041 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:35,041 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:35,044 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:35,044 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:35,045 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2023-09-29T06:19:35,046 DEBUG [hconnection-0x49cb6f05-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:35,048 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49842, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:35,055 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:35,056 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:19:35,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:19:35,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2023-09-29T06:19:35,645 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:36,462 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033029 expired clientIPAndPort=172.31.12.81:36742, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:19:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 11 bytes 2023-09-29T06:19:37,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:19:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.31.12.81:49842 deadline: 1695968378058, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:19:37,072 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:19:37,072 DEBUG [Listener at localhost/43419 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033083 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:19:37,074 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2017 2023-09-29T06:19:37,074 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6c6bdf58 to 127.0.0.1:62866 2023-09-29T06:19:37,074 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:37,074 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:37,075 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:37,075 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:37,075 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x17afbb8b to 127.0.0.1:62866 2023-09-29T06:19:37,086 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=216 (was 216), OpenFileDescriptor=487 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=33 (was 36), ProcessCount=167 (was 167), AvailableMemoryMB=1525 (was 1531) 2023-09-29T06:19:37,096 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=216, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=33, ProcessCount=167, AvailableMemoryMB=1524 2023-09-29T06:19:37,097 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x58e67e9a to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:37,101 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x58e67e9a {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9ed8b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:37,102 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6d4bf255 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:37,105 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca8fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:37,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2023-09-29T06:19:37,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:37,109 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:37,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2023-09-29T06:19:37,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:19:37,202 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2023-09-29T06:19:37,521 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 900e65ae3ca33070283ae33e92a0d0a7, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:37,932 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:37,932 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 900e65ae3ca33070283ae33e92a0d0a7, disabling compactions & flushes 2023-09-29T06:19:37,932 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:37,932 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:37,932 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. after waiting 0 ms 2023-09-29T06:19:37,932 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:37,932 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:37,932 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 900e65ae3ca33070283ae33e92a0d0a7: 2023-09-29T06:19:37,934 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:37,934 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1695968377934"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968377934"}]},"ts":"1695968377934"} 2023-09-29T06:19:37,936 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:37,937 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:37,937 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968377937"}]},"ts":"1695968377937"} 2023-09-29T06:19:37,938 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:37,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=900e65ae3ca33070283ae33e92a0d0a7, ASSIGN}] 2023-09-29T06:19:37,943 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=900e65ae3ca33070283ae33e92a0d0a7, ASSIGN 2023-09-29T06:19:37,944 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=900e65ae3ca33070283ae33e92a0d0a7, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:38,095 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=900e65ae3ca33070283ae33e92a0d0a7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:38,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 900e65ae3ca33070283ae33e92a0d0a7, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:19:38,252 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:38,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7254): Opening region: {ENCODED => 900e65ae3ca33070283ae33e92a0d0a7, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:38,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:38,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7296): checking encryption for 900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7299): checking classloading for 900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,254 INFO [StoreOpener-900e65ae3ca33070283ae33e92a0d0a7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,256 INFO [StoreOpener-900e65ae3ca33070283ae33e92a0d0a7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 900e65ae3ca33070283ae33e92a0d0a7 columnFamilyName testFamily 2023-09-29T06:19:38,256 DEBUG [StoreOpener-900e65ae3ca33070283ae33e92a0d0a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:38,256 INFO [StoreOpener-900e65ae3ca33070283ae33e92a0d0a7-1 {}] regionserver.HStore(324): Store=900e65ae3ca33070283ae33e92a0d0a7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:38,257 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,258 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,260 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1076): writing seq id for 900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:19:38,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:38,263 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1093): Opened 900e65ae3ca33070283ae33e92a0d0a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66031025, jitterRate=-0.016061052680015564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:38,263 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(992): Region open journal for 900e65ae3ca33070283ae33e92a0d0a7: 2023-09-29T06:19:38,264 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7., pid=68, masterSystemTime=1695968378248 2023-09-29T06:19:38,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:38,265 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:19:38,266 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=900e65ae3ca33070283ae33e92a0d0a7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:38,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=68, resume processing ppid=67 2023-09-29T06:19:38,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 900e65ae3ca33070283ae33e92a0d0a7, server=jenkins-hbase3.apache.org,43651,1695968188964 in 170 msec 2023-09-29T06:19:38,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-09-29T06:19:38,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=900e65ae3ca33070283ae33e92a0d0a7, ASSIGN in 327 msec 2023-09-29T06:19:38,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:38,271 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968378271"}]},"ts":"1695968378271"} 2023-09-29T06:19:38,272 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:38,274 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:38,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1670 sec 2023-09-29T06:19:38,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2023-09-29T06:19:38,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:38,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2023-09-29T06:19:38,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:38,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2023-09-29T06:19:38,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:19:40,114 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2023-09-29T06:19:40,114 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:40,114 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:40,117 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:40,118 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:40,118 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2023-09-29T06:19:40,119 DEBUG [hconnection-0x1b3b55a2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:40,121 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49846, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:40,128 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:40,130 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:19:40,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:19:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 88 bytes 2023-09-29T06:19:42,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:19:43,142 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:19:43,142 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-09-29T06:19:43.142Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5027: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:19:43,143 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3013 2023-09-29T06:19:43,143 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6d4bf255 to 127.0.0.1:62866 2023-09-29T06:19:43,143 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:43,144 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:43,144 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:43,144 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:43,144 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x58e67e9a to 127.0.0.1:62866 2023-09-29T06:19:43,155 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=214 (was 216), OpenFileDescriptor=486 (was 487), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=31 (was 33), ProcessCount=167 (was 167), AvailableMemoryMB=1524 (was 1524) 2023-09-29T06:19:43,164 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=214, OpenFileDescriptor=486, MaxFileDescriptor=60000, SystemLoadAverage=31, ProcessCount=167, AvailableMemoryMB=1524 2023-09-29T06:19:43,166 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4cda9e6e to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:43,169 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x4cda9e6e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7e808e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:43,170 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ef56cf0 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:43,174 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21805119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:43,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2023-09-29T06:19:43,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:43,178 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:43,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2023-09-29T06:19:43,179 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-29T06:19:43,202 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2023-09-29T06:19:43,590 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3f1ecb988b69066f325faf08abfe7b96, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:43,647 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:19:43,754 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:44,002 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:44,002 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 3f1ecb988b69066f325faf08abfe7b96, disabling compactions & flushes 2023-09-29T06:19:44,002 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,002 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,002 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. after waiting 0 ms 2023-09-29T06:19:44,002 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,002 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,002 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3f1ecb988b69066f325faf08abfe7b96: 2023-09-29T06:19:44,003 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:44,003 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1695968384003"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968384003"}]},"ts":"1695968384003"} 2023-09-29T06:19:44,005 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:44,005 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:44,006 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968384006"}]},"ts":"1695968384006"} 2023-09-29T06:19:44,007 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:44,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=3f1ecb988b69066f325faf08abfe7b96, ASSIGN}] 2023-09-29T06:19:44,011 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=3f1ecb988b69066f325faf08abfe7b96, ASSIGN 2023-09-29T06:19:44,012 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=3f1ecb988b69066f325faf08abfe7b96, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:44,163 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=3f1ecb988b69066f325faf08abfe7b96, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:44,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure 3f1ecb988b69066f325faf08abfe7b96, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-29T06:19:44,320 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7254): Opening region: {ENCODED => 3f1ecb988b69066f325faf08abfe7b96, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:44,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true 3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:44,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7296): checking encryption for 3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7299): checking classloading for 3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,323 INFO [StoreOpener-3f1ecb988b69066f325faf08abfe7b96-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,324 INFO [StoreOpener-3f1ecb988b69066f325faf08abfe7b96-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f1ecb988b69066f325faf08abfe7b96 columnFamilyName testFamily 2023-09-29T06:19:44,324 DEBUG [StoreOpener-3f1ecb988b69066f325faf08abfe7b96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:44,325 INFO [StoreOpener-3f1ecb988b69066f325faf08abfe7b96-1 {}] regionserver.HStore(324): Store=3f1ecb988b69066f325faf08abfe7b96/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:44,325 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,326 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1076): writing seq id for 3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:19:44,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:44,330 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1093): Opened 3f1ecb988b69066f325faf08abfe7b96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58852634, jitterRate=-0.12302741408348083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:44,331 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(992): Region open journal for 3f1ecb988b69066f325faf08abfe7b96: 2023-09-29T06:19:44,332 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96., pid=71, masterSystemTime=1695968384316 2023-09-29T06:19:44,333 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,333 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:19:44,334 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=3f1ecb988b69066f325faf08abfe7b96, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:44,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-09-29T06:19:44,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure 3f1ecb988b69066f325faf08abfe7b96, server=jenkins-hbase3.apache.org,43651,1695968188964 in 170 msec 2023-09-29T06:19:44,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-09-29T06:19:44,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=3f1ecb988b69066f325faf08abfe7b96, ASSIGN in 326 msec 2023-09-29T06:19:44,338 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:44,338 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968384338"}]},"ts":"1695968384338"} 2023-09-29T06:19:44,340 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:44,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:44,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1660 sec 2023-09-29T06:19:46,069 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033035 expired clientIPAndPort=172.31.12.81:55856, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:19:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-29T06:19:46,182 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2023-09-29T06:19:46,183 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:46,183 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:46,186 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:46,187 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:46,187 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2023-09-29T06:19:46,189 DEBUG [hconnection-0x3d84d2a4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:46,190 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:46,197 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:46,199 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:19:46,199 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:19:46,200 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x4cda9e6e {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:19:46,201 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x4cda9e6e {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:46,202 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54114, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:19:46,205 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:19:46,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033095 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:46,206 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:19:46,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033095 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:46,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2023-09-29T06:19:48,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2023-09-29T06:19:48,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:49,203 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2023-09-29T06:19:49,211 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3005 2023-09-29T06:19:49,212 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ef56cf0 to 127.0.0.1:62866 2023-09-29T06:19:49,212 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033095 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:49,212 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:49,212 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:49,212 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:49,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101033095' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:19:49,212 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:49,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.31.12.81:54114 deadline: 1695968390211, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4693677206101033095' does not exist 2023-09-29T06:19:49,212 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4cda9e6e to 127.0.0.1:62866 2023-09-29T06:19:49,214 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:19:49,212 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,43651,1695968188964 for closing scanner id = 4693677206101033095 for 3f1ecb988b69066f325faf08abfe7b96 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-09-29T06:19:49,224 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=214 (was 214), OpenFileDescriptor=483 (was 486), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=28 (was 31), ProcessCount=167 (was 167), AvailableMemoryMB=1522 (was 1524) 2023-09-29T06:19:49,234 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=214, OpenFileDescriptor=483, MaxFileDescriptor=60000, SystemLoadAverage=28, ProcessCount=167, AvailableMemoryMB=1521 2023-09-29T06:19:49,235 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x454e3f4e to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:49,239 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x454e3f4e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3d76a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:49,240 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x207f694c to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:49,243 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80417d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:49,245 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:49,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2023-09-29T06:19:49,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:49,248 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:49,248 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2023-09-29T06:19:49,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-29T06:19:49,660 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 324b991aff3b2e128bb29836cc661a7c, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:49,707 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:19:49,823 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:50,069 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:50,069 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 324b991aff3b2e128bb29836cc661a7c, disabling compactions & flushes 2023-09-29T06:19:50,069 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,069 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,069 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. after waiting 0 ms 2023-09-29T06:19:50,069 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,069 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,069 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 324b991aff3b2e128bb29836cc661a7c: 2023-09-29T06:19:50,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:50,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1695968390071"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968390071"}]},"ts":"1695968390071"} 2023-09-29T06:19:50,072 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:50,073 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:50,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968390073"}]},"ts":"1695968390073"} 2023-09-29T06:19:50,074 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:50,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=324b991aff3b2e128bb29836cc661a7c, ASSIGN}] 2023-09-29T06:19:50,081 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=324b991aff3b2e128bb29836cc661a7c, ASSIGN 2023-09-29T06:19:50,081 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=324b991aff3b2e128bb29836cc661a7c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:50,232 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=324b991aff3b2e128bb29836cc661a7c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:50,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 324b991aff3b2e128bb29836cc661a7c, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-29T06:19:50,390 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => 324b991aff3b2e128bb29836cc661a7c, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:50,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:50,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for 324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for 324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,392 INFO [StoreOpener-324b991aff3b2e128bb29836cc661a7c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,394 INFO [StoreOpener-324b991aff3b2e128bb29836cc661a7c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 324b991aff3b2e128bb29836cc661a7c columnFamilyName testFamily 2023-09-29T06:19:50,394 DEBUG [StoreOpener-324b991aff3b2e128bb29836cc661a7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:50,394 INFO [StoreOpener-324b991aff3b2e128bb29836cc661a7c-1 {}] regionserver.HStore(324): Store=324b991aff3b2e128bb29836cc661a7c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:50,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,398 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for 324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:19:50,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:50,401 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1093): Opened 324b991aff3b2e128bb29836cc661a7c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74568955, jitterRate=0.11116401851177216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:50,401 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(992): Region open journal for 324b991aff3b2e128bb29836cc661a7c: 2023-09-29T06:19:50,402 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c., pid=74, masterSystemTime=1695968390386 2023-09-29T06:19:50,404 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,404 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:19:50,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=324b991aff3b2e128bb29836cc661a7c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:50,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-09-29T06:19:50,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 324b991aff3b2e128bb29836cc661a7c, server=jenkins-hbase3.apache.org,43651,1695968188964 in 171 msec 2023-09-29T06:19:50,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-09-29T06:19:50,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=324b991aff3b2e128bb29836cc661a7c, ASSIGN in 328 msec 2023-09-29T06:19:50,409 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:50,409 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968390409"}]},"ts":"1695968390409"} 2023-09-29T06:19:50,410 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:50,412 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:50,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1670 sec 2023-09-29T06:19:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-29T06:19:52,252 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2023-09-29T06:19:52,252 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:19:52,252 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:52,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32809 bytes) of info 2023-09-29T06:19:52,258 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033096 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:19:52,259 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2023-09-29T06:19:52,260 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:19:52,260 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2023-09-29T06:19:52,262 DEBUG [hconnection-0x44776e0d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:19:52,263 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35720, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:19:52,269 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:19:52,271 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:19:52,272 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033099 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:19:52,272 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2023-09-29T06:19:53,075 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033037 expired clientIPAndPort=172.31.12.81:55856, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:19:55,204 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2023-09-29T06:19:55,893 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:19:56,177 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033042 expired clientIPAndPort=172.31.12.81:60226, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:19:58,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2023-09-29T06:19:58,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:19:58,773 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2023-09-29T06:19:58,773 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x207f694c to 127.0.0.1:62866 2023-09-29T06:19:58,773 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:58,773 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:19:58,773 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:19:58,774 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:19:58,774 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x454e3f4e to 127.0.0.1:62866 2023-09-29T06:19:58,786 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=214 (was 214), OpenFileDescriptor=482 (was 483), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=24 (was 28), ProcessCount=167 (was 167), AvailableMemoryMB=1513 (was 1521) 2023-09-29T06:19:58,796 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=24, ProcessCount=167, AvailableMemoryMB=1512 2023-09-29T06:19:58,798 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x162f7013 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:58,802 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x162f7013 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e7304f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:58,802 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x36a03c98 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:19:58,805 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7428cc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:19:58,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:19:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2023-09-29T06:19:58,809 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:19:58,809 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:58,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2023-09-29T06:19:58,810 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:19:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:19:59,221 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 630a4ab14136bb0fd608bac65d43a423, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:19:59,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:59,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 630a4ab14136bb0fd608bac65d43a423, disabling compactions & flushes 2023-09-29T06:19:59,633 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. after waiting 0 ms 2023-09-29T06:19:59,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,633 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 630a4ab14136bb0fd608bac65d43a423: 2023-09-29T06:19:59,634 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:19:59,634 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1695968399634"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968399634"}]},"ts":"1695968399634"} 2023-09-29T06:19:59,636 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:19:59,637 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:19:59,637 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968399637"}]},"ts":"1695968399637"} 2023-09-29T06:19:59,638 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:19:59,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=630a4ab14136bb0fd608bac65d43a423, ASSIGN}] 2023-09-29T06:19:59,645 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=630a4ab14136bb0fd608bac65d43a423, ASSIGN 2023-09-29T06:19:59,645 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=630a4ab14136bb0fd608bac65d43a423, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:19:59,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=630a4ab14136bb0fd608bac65d43a423, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:59,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 630a4ab14136bb0fd608bac65d43a423, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:19:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:19:59,954 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,955 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7254): Opening region: {ENCODED => 630a4ab14136bb0fd608bac65d43a423, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:19:59,955 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,955 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:19:59,955 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7296): checking encryption for 630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,955 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7299): checking classloading for 630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,956 INFO [StoreOpener-630a4ab14136bb0fd608bac65d43a423-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,958 INFO [StoreOpener-630a4ab14136bb0fd608bac65d43a423-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 630a4ab14136bb0fd608bac65d43a423 columnFamilyName testFamily 2023-09-29T06:19:59,958 DEBUG [StoreOpener-630a4ab14136bb0fd608bac65d43a423-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:19:59,958 INFO [StoreOpener-630a4ab14136bb0fd608bac65d43a423-1 {}] regionserver.HStore(324): Store=630a4ab14136bb0fd608bac65d43a423/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:19:59,959 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,960 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,962 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1076): writing seq id for 630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:19:59,964 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:19:59,964 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): Opened 630a4ab14136bb0fd608bac65d43a423; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69561780, jitterRate=0.03655129671096802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:19:59,965 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(992): Region open journal for 630a4ab14136bb0fd608bac65d43a423: 2023-09-29T06:19:59,966 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423., pid=77, masterSystemTime=1695968399950 2023-09-29T06:19:59,967 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,967 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:19:59,968 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=630a4ab14136bb0fd608bac65d43a423, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:19:59,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-09-29T06:19:59,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 630a4ab14136bb0fd608bac65d43a423, server=jenkins-hbase3.apache.org,43651,1695968188964 in 171 msec 2023-09-29T06:19:59,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-09-29T06:19:59,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=630a4ab14136bb0fd608bac65d43a423, ASSIGN in 328 msec 2023-09-29T06:19:59,973 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:19:59,973 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968399973"}]},"ts":"1695968399973"} 2023-09-29T06:19:59,975 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:19:59,977 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:19:59,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1700 sec 2023-09-29T06:20:00,281 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033047 expired clientIPAndPort=172.31.12.81:60540, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:20:01,205 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2023-09-29T06:20:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:20:01,814 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2023-09-29T06:20:01,814 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:20:01,814 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:01,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32886 bytes) of info 2023-09-29T06:20:01,817 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033100 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:01,818 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2023-09-29T06:20:01,819 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:01,819 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2023-09-29T06:20:01,821 DEBUG [hconnection-0x5a0f6515-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:01,822 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33276, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:01,829 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:20:01,831 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:20:01,832 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033103 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-09-29T06:20:01,833 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2023-09-29T06:20:02,282 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033049 expired clientIPAndPort=172.31.12.81:60540, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:20:03,333 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2023-09-29T06:20:03,334 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x36a03c98 to 127.0.0.1:62866 2023-09-29T06:20:03,334 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:03,334 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:20:03,334 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:20:03,334 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:03,334 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x162f7013 to 127.0.0.1:62866 2023-09-29T06:20:03,346 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=217 (was 214) - Thread LEAK? -, OpenFileDescriptor=484 (was 482) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=22 (was 24), ProcessCount=167 (was 167), AvailableMemoryMB=1511 (was 1512) 2023-09-29T06:20:03,356 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=217, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=22, ProcessCount=167, AvailableMemoryMB=1510 2023-09-29T06:20:03,357 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e3c3e26 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:03,362 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x0e3c3e26 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60089d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:03,363 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6e0c3a93 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:03,367 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79cf9130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:03,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:20:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2023-09-29T06:20:03,371 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:20:03,371 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:03,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2023-09-29T06:20:03,371 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:20:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-29T06:20:03,783 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ca05f8d8c61d1239597333e521f0b141, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:20:04,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:04,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing ca05f8d8c61d1239597333e521f0b141, disabling compactions & flushes 2023-09-29T06:20:04,193 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. after waiting 0 ms 2023-09-29T06:20:04,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,193 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for ca05f8d8c61d1239597333e521f0b141: 2023-09-29T06:20:04,194 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:20:04,195 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1695968404194"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968404194"}]},"ts":"1695968404194"} 2023-09-29T06:20:04,196 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:20:04,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:20:04,197 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968404197"}]},"ts":"1695968404197"} 2023-09-29T06:20:04,198 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:20:04,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ca05f8d8c61d1239597333e521f0b141, ASSIGN}] 2023-09-29T06:20:04,203 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ca05f8d8c61d1239597333e521f0b141, ASSIGN 2023-09-29T06:20:04,203 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ca05f8d8c61d1239597333e521f0b141, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:20:04,354 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=ca05f8d8c61d1239597333e521f0b141, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:04,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure ca05f8d8c61d1239597333e521f0b141, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:20:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-29T06:20:04,513 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,513 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7254): Opening region: {ENCODED => ca05f8d8c61d1239597333e521f0b141, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:20:04,514 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,514 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:04,514 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7296): checking encryption for ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,514 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7299): checking classloading for ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,515 INFO [StoreOpener-ca05f8d8c61d1239597333e521f0b141-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,517 INFO [StoreOpener-ca05f8d8c61d1239597333e521f0b141-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca05f8d8c61d1239597333e521f0b141 columnFamilyName testFamily 2023-09-29T06:20:04,517 DEBUG [StoreOpener-ca05f8d8c61d1239597333e521f0b141-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:04,517 INFO [StoreOpener-ca05f8d8c61d1239597333e521f0b141-1 {}] regionserver.HStore(324): Store=ca05f8d8c61d1239597333e521f0b141/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:20:04,518 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,519 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1076): writing seq id for ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:04,523 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:20:04,524 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): Opened ca05f8d8c61d1239597333e521f0b141; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69382383, jitterRate=0.03387807309627533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:20:04,524 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(992): Region open journal for ca05f8d8c61d1239597333e521f0b141: 2023-09-29T06:20:04,525 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141., pid=80, masterSystemTime=1695968404508 2023-09-29T06:20:04,527 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,527 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:04,527 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=ca05f8d8c61d1239597333e521f0b141, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:04,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-09-29T06:20:04,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure ca05f8d8c61d1239597333e521f0b141, server=jenkins-hbase3.apache.org,43651,1695968188964 in 173 msec 2023-09-29T06:20:04,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=79, resume processing ppid=78 2023-09-29T06:20:04,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=ca05f8d8c61d1239597333e521f0b141, ASSIGN in 328 msec 2023-09-29T06:20:04,532 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:20:04,532 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968404532"}]},"ts":"1695968404532"} 2023-09-29T06:20:04,534 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:20:04,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:20:04,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1680 sec 2023-09-29T06:20:05,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:20:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-29T06:20:06,375 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2023-09-29T06:20:06,375 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:20:06,375 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:06,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32859 bytes) of info 2023-09-29T06:20:06,379 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033104 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:06,380 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2023-09-29T06:20:06,380 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:06,380 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2023-09-29T06:20:06,382 DEBUG [hconnection-0x7ef988c2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:06,384 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33288, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:06,391 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:20:06,393 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:20:06,394 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:20:06,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033107 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:06,395 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:20:06,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033107 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:06,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2023-09-29T06:20:07,206 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2023-09-29T06:20:08,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2023-09-29T06:20:08,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:20:08,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2023-09-29T06:20:08,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:20:09,403 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4008: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4008: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:20:09,403 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3008 2023-09-29T06:20:09,403 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6e0c3a93 to 127.0.0.1:62866 2023-09-29T06:20:09,404 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:09,404 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:20:09,404 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:20:09,404 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:09,404 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e3c3e26 to 127.0.0.1:62866 2023-09-29T06:20:09,415 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=214 (was 217), OpenFileDescriptor=485 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=20 (was 22), ProcessCount=167 (was 167), AvailableMemoryMB=1472 (was 1510) 2023-09-29T06:20:09,425 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=214, OpenFileDescriptor=485, MaxFileDescriptor=60000, SystemLoadAverage=20, ProcessCount=167, AvailableMemoryMB=1471 2023-09-29T06:20:09,426 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1324db50 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:09,433 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x1324db50 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a5a405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:09,434 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5f998f09 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:09,437 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88130ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:09,439 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:20:09,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2023-09-29T06:20:09,441 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:20:09,441 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:09,441 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2023-09-29T06:20:09,442 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:20:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-29T06:20:09,856 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 66db2bdb534f53f2003060ce8de1b2ac, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:20:09,896 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:20:10,267 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:10,267 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 66db2bdb534f53f2003060ce8de1b2ac, disabling compactions & flushes 2023-09-29T06:20:10,267 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,267 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,267 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. after waiting 0 ms 2023-09-29T06:20:10,267 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,267 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,267 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 66db2bdb534f53f2003060ce8de1b2ac: 2023-09-29T06:20:10,268 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:20:10,268 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1695968410268"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968410268"}]},"ts":"1695968410268"} 2023-09-29T06:20:10,270 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:20:10,270 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:20:10,271 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968410270"}]},"ts":"1695968410270"} 2023-09-29T06:20:10,272 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:20:10,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66db2bdb534f53f2003060ce8de1b2ac, ASSIGN}] 2023-09-29T06:20:10,276 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66db2bdb534f53f2003060ce8de1b2ac, ASSIGN 2023-09-29T06:20:10,277 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66db2bdb534f53f2003060ce8de1b2ac, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:20:10,428 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=66db2bdb534f53f2003060ce8de1b2ac, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:10,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 66db2bdb534f53f2003060ce8de1b2ac, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:20:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-29T06:20:10,584 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 66db2bdb534f53f2003060ce8de1b2ac, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:20:10,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:10,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,585 INFO [StoreOpener-66db2bdb534f53f2003060ce8de1b2ac-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,587 INFO [StoreOpener-66db2bdb534f53f2003060ce8de1b2ac-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66db2bdb534f53f2003060ce8de1b2ac columnFamilyName testFamily 2023-09-29T06:20:10,587 DEBUG [StoreOpener-66db2bdb534f53f2003060ce8de1b2ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:10,588 INFO [StoreOpener-66db2bdb534f53f2003060ce8de1b2ac-1 {}] regionserver.HStore(324): Store=66db2bdb534f53f2003060ce8de1b2ac/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:20:10,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:10,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:20:10,594 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1093): Opened 66db2bdb534f53f2003060ce8de1b2ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72289942, jitterRate=0.07720407843589783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:20:10,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 66db2bdb534f53f2003060ce8de1b2ac: 2023-09-29T06:20:10,595 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac., pid=83, masterSystemTime=1695968410580 2023-09-29T06:20:10,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,596 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:20:10,597 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=66db2bdb534f53f2003060ce8de1b2ac, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:10,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-09-29T06:20:10,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 66db2bdb534f53f2003060ce8de1b2ac, server=jenkins-hbase3.apache.org,43651,1695968188964 in 169 msec 2023-09-29T06:20:10,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-09-29T06:20:10,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66db2bdb534f53f2003060ce8de1b2ac, ASSIGN in 324 msec 2023-09-29T06:20:10,601 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:20:10,601 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968410601"}]},"ts":"1695968410601"} 2023-09-29T06:20:10,602 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:20:10,605 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:20:10,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1660 sec 2023-09-29T06:20:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-29T06:20:12,445 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2023-09-29T06:20:12,445 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:20:12,445 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:12,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32913 bytes) of info 2023-09-29T06:20:12,449 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033108 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:12,451 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2023-09-29T06:20:12,451 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:12,451 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2023-09-29T06:20:12,454 DEBUG [hconnection-0x201059c7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:12,455 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42320, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:12,463 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:20:12,465 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:20:12,466 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:20:12,467 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2023-09-29T06:20:13,207 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2023-09-29T06:20:14,413 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033063 expired clientIPAndPort=172.31.12.81:39834, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:16,123 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:20:18,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2023-09-29T06:20:18,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:20:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 11 bytes 2023-09-29T06:20:19,483 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:20:19,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:42320 deadline: 1695968421467, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-09-29T06:20:19,484 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-09-29T06:20:19,485 DEBUG [Listener at localhost/43419 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 4693677206101033111 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-09-29T06:20:19,486 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7021 2023-09-29T06:20:19,487 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5f998f09 to 127.0.0.1:62866 2023-09-29T06:20:19,487 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:19,487 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:20:19,487 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:20:19,487 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:19,487 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1324db50 to 127.0.0.1:62866 2023-09-29T06:20:19,498 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=214 (was 214), OpenFileDescriptor=482 (was 485), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=17 (was 20), ProcessCount=168 (was 167) - ProcessCount LEAK? -, AvailableMemoryMB=1468 (was 1471) 2023-09-29T06:20:19,508 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=17, ProcessCount=168, AvailableMemoryMB=1468 2023-09-29T06:20:19,509 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e09f00d to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:19,513 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x0e09f00d {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a60c654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:19,513 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x74f532c3 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:19,518 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610e23cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:19,520 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:20:19,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2023-09-29T06:20:19,522 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:20:19,522 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:19,522 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2023-09-29T06:20:19,522 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:20:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=84 2023-09-29T06:20:19,936 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 540f2380dc3488a31274410004f1161b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:20:20,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:20,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 540f2380dc3488a31274410004f1161b, disabling compactions & flushes 2023-09-29T06:20:20,346 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. after waiting 0 ms 2023-09-29T06:20:20,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,346 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 540f2380dc3488a31274410004f1161b: 2023-09-29T06:20:20,347 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:20:20,347 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1695968420347"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968420347"}]},"ts":"1695968420347"} 2023-09-29T06:20:20,349 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:20:20,349 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:20:20,350 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968420350"}]},"ts":"1695968420350"} 2023-09-29T06:20:20,351 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:20:20,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=540f2380dc3488a31274410004f1161b, ASSIGN}] 2023-09-29T06:20:20,355 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=540f2380dc3488a31274410004f1161b, ASSIGN 2023-09-29T06:20:20,356 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=540f2380dc3488a31274410004f1161b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:20:20,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=540f2380dc3488a31274410004f1161b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:20,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure 540f2380dc3488a31274410004f1161b, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:20:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=84 2023-09-29T06:20:20,667 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7254): Opening region: {ENCODED => 540f2380dc3488a31274410004f1161b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:20:20,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true 540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:20,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7296): checking encryption for 540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7299): checking classloading for 540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,669 INFO [StoreOpener-540f2380dc3488a31274410004f1161b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,671 INFO [StoreOpener-540f2380dc3488a31274410004f1161b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 540f2380dc3488a31274410004f1161b columnFamilyName testFamily 2023-09-29T06:20:20,671 DEBUG [StoreOpener-540f2380dc3488a31274410004f1161b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:20,672 INFO [StoreOpener-540f2380dc3488a31274410004f1161b-1 {}] regionserver.HStore(324): Store=540f2380dc3488a31274410004f1161b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:20:20,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,673 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1076): writing seq id for 540f2380dc3488a31274410004f1161b 2023-09-29T06:20:20,677 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:20:20,678 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1093): Opened 540f2380dc3488a31274410004f1161b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68942367, jitterRate=0.02732132375240326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:20:20,678 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(992): Region open journal for 540f2380dc3488a31274410004f1161b: 2023-09-29T06:20:20,679 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., pid=86, masterSystemTime=1695968420660 2023-09-29T06:20:20,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,681 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:20,681 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=540f2380dc3488a31274410004f1161b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:20,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=86, resume processing ppid=85 2023-09-29T06:20:20,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure 540f2380dc3488a31274410004f1161b, server=jenkins-hbase3.apache.org,43651,1695968188964 in 173 msec 2023-09-29T06:20:20,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=85, resume processing ppid=84 2023-09-29T06:20:20,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=540f2380dc3488a31274410004f1161b, ASSIGN in 330 msec 2023-09-29T06:20:20,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:20:20,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968420686"}]},"ts":"1695968420686"} 2023-09-29T06:20:20,687 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:20:20,690 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:20:20,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1700 sec 2023-09-29T06:20:21,418 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033065 expired clientIPAndPort=172.31.12.81:39834, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=84 2023-09-29T06:20:22,526 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2023-09-29T06:20:22,526 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:20:22,526 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:22,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32913 bytes) of info 2023-09-29T06:20:22,530 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033113 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:22,531 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2023-09-29T06:20:22,531 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:22,531 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2023-09-29T06:20:22,533 DEBUG [hconnection-0x5b329ce6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:22,534 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34652, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:22,541 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:20:22,543 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:20:22,543 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:20:22,544 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x0e09f00d {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:20:22,544 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x0e09f00d {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:22,545 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34662, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:20:22,548 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:20:25,208 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2023-09-29T06:20:26,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:20:28,550 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2023-09-29T06:20:28,551 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2023-09-29T06:20:28,551 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:20:28,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2023-09-29T06:20:28,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:20:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 88 bytes 2023-09-29T06:20:29,424 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033070 expired clientIPAndPort=172.31.12.81:47780, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:20:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:20:29,567 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-09-29T06:20:31,550 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms 2023-09-29T06:20:31,550 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms 2023-09-29T06:20:31,550 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:20:31,551 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-09-29T06:20:28.551Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2023-09-29T06:20:31.550Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1984ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-09-29T06:20:31,553 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9010 2023-09-29T06:20:31,553 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x74f532c3 to 127.0.0.1:62866 2023-09-29T06:20:31,553 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:31,553 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:20:31,554 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:20:31,554 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:31,554 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e09f00d to 127.0.0.1:62866 2023-09-29T06:20:31,566 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=214 (was 214), OpenFileDescriptor=482 (was 482), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=14 (was 17), ProcessCount=168 (was 168), AvailableMemoryMB=1466 (was 1468) 2023-09-29T06:20:31,575 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=14, ProcessCount=168, AvailableMemoryMB=1466 2023-09-29T06:20:31,576 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x16bb452a to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:31,580 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x16bb452a {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28282eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:31,581 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x24f3f8a1 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:31,584 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e53e795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:31,586 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:20:31,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2023-09-29T06:20:31,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:20:31,587 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:31,588 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2023-09-29T06:20:31,588 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:20:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:20:31,999 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2e59844a5245f32887e00ecd6717b11e, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:20:32,026 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033077 expired clientIPAndPort=172.31.12.81:49836, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:32,027 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033074 expired clientIPAndPort=172.31.12.81:49836, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:32,027 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033076 expired clientIPAndPort=172.31.12.81:49836, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:32,409 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:32,409 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 2e59844a5245f32887e00ecd6717b11e, disabling compactions & flushes 2023-09-29T06:20:32,409 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,409 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,409 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. after waiting 0 ms 2023-09-29T06:20:32,409 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,409 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,409 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2e59844a5245f32887e00ecd6717b11e: 2023-09-29T06:20:32,410 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:20:32,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1695968432410"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968432410"}]},"ts":"1695968432410"} 2023-09-29T06:20:32,412 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:20:32,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:20:32,413 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968432413"}]},"ts":"1695968432413"} 2023-09-29T06:20:32,414 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:20:32,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=2e59844a5245f32887e00ecd6717b11e, ASSIGN}] 2023-09-29T06:20:32,419 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=2e59844a5245f32887e00ecd6717b11e, ASSIGN 2023-09-29T06:20:32,420 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=2e59844a5245f32887e00ecd6717b11e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:20:32,571 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=2e59844a5245f32887e00ecd6717b11e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:32,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure 2e59844a5245f32887e00ecd6717b11e, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:20:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:20:32,730 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7254): Opening region: {ENCODED => 2e59844a5245f32887e00ecd6717b11e, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:20:32,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true 2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:32,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7296): checking encryption for 2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7299): checking classloading for 2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,733 INFO [StoreOpener-2e59844a5245f32887e00ecd6717b11e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,734 INFO [StoreOpener-2e59844a5245f32887e00ecd6717b11e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e59844a5245f32887e00ecd6717b11e columnFamilyName testFamily 2023-09-29T06:20:32,734 DEBUG [StoreOpener-2e59844a5245f32887e00ecd6717b11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:32,734 INFO [StoreOpener-2e59844a5245f32887e00ecd6717b11e-1 {}] regionserver.HStore(324): Store=2e59844a5245f32887e00ecd6717b11e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:20:32,735 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,736 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,738 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1076): writing seq id for 2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:32,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:20:32,741 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1093): Opened 2e59844a5245f32887e00ecd6717b11e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68275883, jitterRate=0.017389938235282898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:20:32,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(992): Region open journal for 2e59844a5245f32887e00ecd6717b11e: 2023-09-29T06:20:32,742 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e., pid=89, masterSystemTime=1695968432725 2023-09-29T06:20:32,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,743 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:32,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=2e59844a5245f32887e00ecd6717b11e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:32,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=89, resume processing ppid=88 2023-09-29T06:20:32,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure 2e59844a5245f32887e00ecd6717b11e, server=jenkins-hbase3.apache.org,43651,1695968188964 in 172 msec 2023-09-29T06:20:32,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-09-29T06:20:32,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=2e59844a5245f32887e00ecd6717b11e, ASSIGN in 328 msec 2023-09-29T06:20:32,748 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:20:32,748 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968432748"}]},"ts":"1695968432748"} 2023-09-29T06:20:32,749 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:20:32,751 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:20:32,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1650 sec 2023-09-29T06:20:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:20:34,592 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2023-09-29T06:20:34,592 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:20:34,592 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:34,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32861 bytes) of info 2023-09-29T06:20:34,595 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033120 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:34,597 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2023-09-29T06:20:34,597 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:34,597 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2023-09-29T06:20:34,599 DEBUG [hconnection-0x1bce6ac9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:34,600 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41176, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:34,607 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:20:34,609 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-09-29T06:20:34,611 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-09-29T06:20:34,611 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033123 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:34,612 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-09-29T06:20:34,614 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033123 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:34,614 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2023-09-29T06:20:36,068 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:20:37,130 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033083 expired clientIPAndPort=172.31.12.81:49842, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:20:37,209 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2023-09-29T06:20:38,272 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:20:38,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2023-09-29T06:20:38,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:20:41,634 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033088 expired clientIPAndPort=172.31.12.81:49846, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:43,623 DEBUG [Listener at localhost/43419 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10016: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9006ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10016: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9006ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9006ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9006ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-09-29T06:20:43,624 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9011 2023-09-29T06:20:43,624 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x24f3f8a1 to 127.0.0.1:62866 2023-09-29T06:20:43,624 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:43,624 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:20:43,624 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:20:43,624 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:43,624 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x16bb452a to 127.0.0.1:62866 2023-09-29T06:20:43,635 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=212 (was 214), OpenFileDescriptor=478 (was 482), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=11 (was 14), ProcessCount=168 (was 168), AvailableMemoryMB=1460 (was 1466) 2023-09-29T06:20:43,645 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=212, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=11, ProcessCount=168, AvailableMemoryMB=1460 2023-09-29T06:20:43,646 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5d2fbad7 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:43,650 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x5d2fbad7 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebe66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:43,650 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1a598864 to 127.0.0.1:62866 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:20:43,653 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5548d098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:20:43,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:20:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] procedure2.ProcedureExecutor(1032): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2023-09-29T06:20:43,657 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:20:43,657 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:43,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2023-09-29T06:20:43,658 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:20:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=90 2023-09-29T06:20:43,736 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033090 expired clientIPAndPort=172.31.12.81:49846, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:44,071 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a968f54380ef9c5e30892a9639d27324, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606 2023-09-29T06:20:44,115 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:20:44,480 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:44,480 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing a968f54380ef9c5e30892a9639d27324, disabling compactions & flushes 2023-09-29T06:20:44,480 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,480 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,481 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. after waiting 0 ms 2023-09-29T06:20:44,481 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,481 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,481 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for a968f54380ef9c5e30892a9639d27324: 2023-09-29T06:20:44,482 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:20:44,482 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1695968444482"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968444482"}]},"ts":"1695968444482"} 2023-09-29T06:20:44,484 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:20:44,484 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:20:44,484 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968444484"}]},"ts":"1695968444484"} 2023-09-29T06:20:44,486 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2023-09-29T06:20:44,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=a968f54380ef9c5e30892a9639d27324, ASSIGN}] 2023-09-29T06:20:44,492 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=a968f54380ef9c5e30892a9639d27324, ASSIGN 2023-09-29T06:20:44,493 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=a968f54380ef9c5e30892a9639d27324, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43651,1695968188964; forceNewPlan=false, retain=false 2023-09-29T06:20:44,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=a968f54380ef9c5e30892a9639d27324, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:44,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure a968f54380ef9c5e30892a9639d27324, server=jenkins-hbase3.apache.org,43651,1695968188964}] 2023-09-29T06:20:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=90 2023-09-29T06:20:44,804 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7254): Opening region: {ENCODED => a968f54380ef9c5e30892a9639d27324, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:20:44,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:20:44,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7296): checking encryption for a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7299): checking classloading for a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,806 INFO [StoreOpener-a968f54380ef9c5e30892a9639d27324-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,808 INFO [StoreOpener-a968f54380ef9c5e30892a9639d27324-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a968f54380ef9c5e30892a9639d27324 columnFamilyName testFamily 2023-09-29T06:20:44,808 DEBUG [StoreOpener-a968f54380ef9c5e30892a9639d27324-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:20:44,808 INFO [StoreOpener-a968f54380ef9c5e30892a9639d27324-1 {}] regionserver.HStore(324): Store=a968f54380ef9c5e30892a9639d27324/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:20:44,809 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,810 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,812 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1076): writing seq id for a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:44,815 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:20:44,816 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): Opened a968f54380ef9c5e30892a9639d27324; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61074619, jitterRate=-0.08991725742816925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:20:44,816 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(992): Region open journal for a968f54380ef9c5e30892a9639d27324: 2023-09-29T06:20:44,817 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., pid=92, masterSystemTime=1695968444797 2023-09-29T06:20:44,818 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,818 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:44,819 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=a968f54380ef9c5e30892a9639d27324, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:44,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-09-29T06:20:44,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure a968f54380ef9c5e30892a9639d27324, server=jenkins-hbase3.apache.org,43651,1695968188964 in 175 msec 2023-09-29T06:20:44,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=91, resume processing ppid=90 2023-09-29T06:20:44,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=a968f54380ef9c5e30892a9639d27324, ASSIGN in 330 msec 2023-09-29T06:20:44,823 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:20:44,823 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968444823"}]},"ts":"1695968444823"} 2023-09-29T06:20:44,824 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2023-09-29T06:20:44,826 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:20:44,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1710 sec 2023-09-29T06:20:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38609 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=90 2023-09-29T06:20:46,662 INFO [Listener at localhost/43419 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2023-09-29T06:20:46,662 DEBUG [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2023-09-29T06:20:46,662 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:46,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43651 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32921 bytes) of info 2023-09-29T06:20:46,666 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 4693677206101033124 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-09-29T06:20:46,667 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2023-09-29T06:20:46,668 INFO [Listener at localhost/43419 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-29T06:20:46,668 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2023-09-29T06:20:46,670 DEBUG [hconnection-0x2606fb26-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:46,672 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44398, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:46,679 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-09-29T06:20:46,681 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-09-29T06:20:46,682 DEBUG [Listener at localhost/43419 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-09-29T06:20:46,683 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x5d2fbad7 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=-1] 2023-09-29T06:20:46,683 DEBUG [ReadOnlyZKClient-127.0.0.1:62866@0x5d2fbad7 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:20:46,685 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44410, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:20:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:20:46,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:20:47,690 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2023-09-29T06:20:47,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2023-09-29T06:20:47,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:20:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 88 bytes 2023-09-29T06:20:48,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2023-09-29T06:20:48,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-09-29T06:20:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2] 2023-09-29T06:20:48,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43651 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-09-29T06:20:49,210 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2023-09-29T06:20:49,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=976ms 2023-09-29T06:20:49,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=976ms 2023-09-29T06:20:49,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324., hostname=jenkins-hbase3.apache.org,43651,1695968188964, seqNum=2 from cache 2023-09-29T06:20:49,692 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-09-29T06:20:47.691Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2023-09-29T06:20:49.691Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=976ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:43651 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-09-29T06:20:49,694 INFO [Listener at localhost/43419 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3012 2023-09-29T06:20:49,694 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1a598864 to 127.0.0.1:62866 2023-09-29T06:20:49,694 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:49,694 INFO [Listener at localhost/43419 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43419. 2023-09-29T06:20:49,694 DEBUG [Listener at localhost/43419 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:20:49,694 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:49,694 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5d2fbad7 to 127.0.0.1:62866 2023-09-29T06:20:49,706 INFO [Listener at localhost/43419 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=214 (was 212) - Thread LEAK? -, OpenFileDescriptor=483 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=18 (was 11) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=1454 (was 1460) 2023-09-29T06:20:49,706 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-29T06:20:49,706 INFO [Listener at localhost/43419 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:20:49,706 DEBUG [Listener at localhost/43419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x606f5fd4 to 127.0.0.1:62866 2023-09-29T06:20:49,707 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:49,707 DEBUG [Listener at localhost/43419 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-29T06:20:49,707 DEBUG [Listener at localhost/43419 {}] util.JVMClusterUtil(257): Found active master hash=1616927353, stopped=false 2023-09-29T06:20:49,707 INFO [Listener at localhost/43419 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:20:49,710 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-09-29T06:20:49,710 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-09-29T06:20:49,710 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:20:49,710 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:20:49,710 INFO [Listener at localhost/43419 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-29T06:20:49,710 DEBUG [Listener at localhost/43419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:49,710 INFO [Listener at localhost/43419 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,43651,1695968188964' ***** 2023-09-29T06:20:49,710 INFO [Listener at localhost/43419 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-29T06:20:49,710 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:20:49,710 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:20:49,711 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-29T06:20:49,711 INFO [RS:0;jenkins-hbase3:43651 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-29T06:20:49,711 INFO [RS:0;jenkins-hbase3:43651 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-29T06:20:49,711 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 881582bf9027dda670e449ba9791bddb 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 324b991aff3b2e128bb29836cc661a7c 2023-09-29T06:20:49,712 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for eb4991a9a0ce4f84f517a97a03fff332 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 94654e3c565dd5486a5739c3568d8845 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 630a4ab14136bb0fd608bac65d43a423 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 540f2380dc3488a31274410004f1161b 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for a968f54380ef9c5e30892a9639d27324 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for cb052ffb687b3cbe7ab95c4d907309a7 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 381724feeefef7418ce9cb890fc7b15f 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for c5d82055d13f28d8394959085f7e9b9c 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 144c806451470faf2697726823a29c67 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 3f1ecb988b69066f325faf08abfe7b96 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 2e59844a5245f32887e00ecd6717b11e 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for d2ca20f2151278b7b07397ded57f313b 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for ee80440612009a663af11a4db3cdc824 2023-09-29T06:20:49,712 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 881582bf9027dda670e449ba9791bddb, disabling compactions & flushes 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 900e65ae3ca33070283ae33e92a0d0a7 2023-09-29T06:20:49,712 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:49,712 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:49,713 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 6c193cd68953fa128ebdeb920ceacb28 2023-09-29T06:20:49,713 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. after waiting 0 ms 2023-09-29T06:20:49,713 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 9a7d56d0a85a7d831c743aa9169ba277 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 3b2e2b8af1b0a36351d7765af650b765 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for a5a35f3ea933f9ee2575761d0ca57b8b 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for b60b1ecdd7b636a2e033c7b1d93d483f 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 58d56be7906234b32fb9db25b080a032 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 66db2bdb534f53f2003060ce8de1b2ac 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 320342231d3957a0d6191f34e62ec4a2 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:20:49,713 DEBUG [RS:0;jenkins-hbase3:43651 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-29T06:20:49,713 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 881582bf9027dda670e449ba9791bddb 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:49,713 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 30 regions to close 2023-09-29T06:20:49,714 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-29T06:20:49,714 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-29T06:20:49,714 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-29T06:20:49,714 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-29T06:20:49,714 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-29T06:20:49,714 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=48.69 KB heapSize=75.08 KB 2023-09-29T06:20:49,715 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 881582bf9027dda670e449ba9791bddb, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:49,740 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease 4693677206101033095 expired clientIPAndPort=172.31.12.81:54114, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:20:49,989 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 881582bf9027dda670e449ba9791bddb, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:50,060 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/.tmp/testFamily/f52eac3885884c5b81117c99ed447aa1 is 61, key is row-0/testFamily:testQualifier/1695968281967/Put/seqid=0 2023-09-29T06:20:50,068 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/.tmp/info/4f1904707a484df9827b3a02e248fc66 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b./info:regioninfo/1695968253264/Put/seqid=0 2023-09-29T06:20:50,189 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 881582bf9027dda670e449ba9791bddb, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:50,207 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-09-29T06:20:50,390 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 881582bf9027dda670e449ba9791bddb, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:50,484 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/.tmp/testFamily/f52eac3885884c5b81117c99ed447aa1 2023-09-29T06:20:50,484 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.45 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/.tmp/info/4f1904707a484df9827b3a02e248fc66 2023-09-29T06:20:50,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/.tmp/testFamily/f52eac3885884c5b81117c99ed447aa1 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/testFamily/f52eac3885884c5b81117c99ed447aa1 2023-09-29T06:20:50,550 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/testFamily/f52eac3885884c5b81117c99ed447aa1, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:50,553 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 881582bf9027dda670e449ba9791bddb in 840ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:50,558 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:20:50,560 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/.tmp/table/b190c38ebeca4fc3aa10fc1382a2c016 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1695968253276/Put/seqid=0 2023-09-29T06:20:50,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_0_-false/881582bf9027dda670e449ba9791bddb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:50,563 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:20:50,563 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 881582bf9027dda670e449ba9791bddb: 2023-09-29T06:20:50,563 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1695968278946.881582bf9027dda670e449ba9791bddb. 2023-09-29T06:20:50,563 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 324b991aff3b2e128bb29836cc661a7c, disabling compactions & flushes 2023-09-29T06:20:50,564 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:20:50,564 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:20:50,564 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. after waiting 0 ms 2023-09-29T06:20:50,564 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:20:50,564 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 324b991aff3b2e128bb29836cc661a7c 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:50,584 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/.tmp/testFamily/bfecb55623004bfb9b4188ef39a0fdb5 is 61, key is row-0/testFamily:testQualifier/1695968392265/Put/seqid=0 2023-09-29T06:20:50,591 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:50,612 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-29T06:20:50,612 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-29T06:20:50,791 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 29 regions to close 2023-09-29T06:20:50,791 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:50,977 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/.tmp/table/b190c38ebeca4fc3aa10fc1382a2c016 2023-09-29T06:20:50,985 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/.tmp/info/4f1904707a484df9827b3a02e248fc66 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/info/4f1904707a484df9827b3a02e248fc66 2023-09-29T06:20:50,992 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 1588230740, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 324b991aff3b2e128bb29836cc661a7c, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:50,993 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/info/4f1904707a484df9827b3a02e248fc66, entries=290, sequenceid=149, filesize=51.4 K 2023-09-29T06:20:50,994 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/.tmp/testFamily/bfecb55623004bfb9b4188ef39a0fdb5 2023-09-29T06:20:50,995 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/.tmp/table/b190c38ebeca4fc3aa10fc1382a2c016 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/table/b190c38ebeca4fc3aa10fc1382a2c016 2023-09-29T06:20:51,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/.tmp/testFamily/bfecb55623004bfb9b4188ef39a0fdb5 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/testFamily/bfecb55623004bfb9b4188ef39a0fdb5 2023-09-29T06:20:51,003 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/table/b190c38ebeca4fc3aa10fc1382a2c016, entries=58, sequenceid=149, filesize=9.6 K 2023-09-29T06:20:51,004 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~48.69 KB/49854, heapSize ~74.80 KB/76592, currentSize=0 B/0 for 1588230740 in 1290ms, sequenceid=149, compaction requested=false 2023-09-29T06:20:51,011 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/testFamily/bfecb55623004bfb9b4188ef39a0fdb5, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:51,012 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2023-09-29T06:20:51,012 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 324b991aff3b2e128bb29836cc661a7c in 448ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:51,013 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-29T06:20:51,014 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-29T06:20:51,014 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-29T06:20:51,014 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-29T06:20:51,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_1_-true/324b991aff3b2e128bb29836cc661a7c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:51,018 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:20:51,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 324b991aff3b2e128bb29836cc661a7c: 2023-09-29T06:20:51,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1695968389245.324b991aff3b2e128bb29836cc661a7c. 2023-09-29T06:20:51,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing eb4991a9a0ce4f84f517a97a03fff332, disabling compactions & flushes 2023-09-29T06:20:51,019 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:20:51,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:20:51,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. after waiting 0 ms 2023-09-29T06:20:51,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:20:51,019 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing eb4991a9a0ce4f84f517a97a03fff332 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:51,031 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/.tmp/testFamily/329bc99ca05e45f780d0e3444c99e845 is 61, key is row-0/testFamily:testQualifier/1695968344749/Put/seqid=0 2023-09-29T06:20:51,192 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:51,392 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, eb4991a9a0ce4f84f517a97a03fff332, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:51,438 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/.tmp/testFamily/329bc99ca05e45f780d0e3444c99e845 2023-09-29T06:20:51,446 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/.tmp/testFamily/329bc99ca05e45f780d0e3444c99e845 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/testFamily/329bc99ca05e45f780d0e3444c99e845 2023-09-29T06:20:51,452 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/testFamily/329bc99ca05e45f780d0e3444c99e845, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:51,453 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb4991a9a0ce4f84f517a97a03fff332 in 434ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:51,458 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eb4991a9a0ce4f84f517a97a03fff332/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:51,459 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:20:51,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for eb4991a9a0ce4f84f517a97a03fff332: 2023-09-29T06:20:51,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1695968341730.eb4991a9a0ce4f84f517a97a03fff332. 2023-09-29T06:20:51,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 94654e3c565dd5486a5739c3568d8845, disabling compactions & flushes 2023-09-29T06:20:51,459 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:20:51,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:20:51,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. after waiting 0 ms 2023-09-29T06:20:51,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:20:51,459 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 94654e3c565dd5486a5739c3568d8845 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:51,471 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/.tmp/testFamily/6fb1268d63804178a23972e643fa317e is 61, key is row-0/testFamily:testQualifier/1695968307340/Put/seqid=0 2023-09-29T06:20:51,593 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:51,793 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 26 regions to close 2023-09-29T06:20:51,793 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 94654e3c565dd5486a5739c3568d8845, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:51,879 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/.tmp/testFamily/6fb1268d63804178a23972e643fa317e 2023-09-29T06:20:51,887 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/.tmp/testFamily/6fb1268d63804178a23972e643fa317e as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/testFamily/6fb1268d63804178a23972e643fa317e 2023-09-29T06:20:51,893 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/testFamily/6fb1268d63804178a23972e643fa317e, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:51,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 94654e3c565dd5486a5739c3568d8845 in 435ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:51,899 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_0_-false/94654e3c565dd5486a5739c3568d8845/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:51,900 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:20:51,900 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 94654e3c565dd5486a5739c3568d8845: 2023-09-29T06:20:51,900 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1695968304319.94654e3c565dd5486a5739c3568d8845. 2023-09-29T06:20:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 630a4ab14136bb0fd608bac65d43a423, disabling compactions & flushes 2023-09-29T06:20:51,901 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:20:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:20:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. after waiting 0 ms 2023-09-29T06:20:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:20:51,901 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 630a4ab14136bb0fd608bac65d43a423 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:51,917 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/.tmp/testFamily/c9c444fba8984a76a9503ef843c3b443 is 61, key is row-0/testFamily:testQualifier/1695968401825/Put/seqid=0 2023-09-29T06:20:51,993 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:52,194 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 630a4ab14136bb0fd608bac65d43a423, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:52,325 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/.tmp/testFamily/c9c444fba8984a76a9503ef843c3b443 2023-09-29T06:20:52,332 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/.tmp/testFamily/c9c444fba8984a76a9503ef843c3b443 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/testFamily/c9c444fba8984a76a9503ef843c3b443 2023-09-29T06:20:52,338 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/testFamily/c9c444fba8984a76a9503ef843c3b443, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:52,340 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 630a4ab14136bb0fd608bac65d43a423 in 439ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:52,346 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_1_-true/630a4ab14136bb0fd608bac65d43a423/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:52,347 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:20:52,347 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 630a4ab14136bb0fd608bac65d43a423: 2023-09-29T06:20:52,347 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1695968398807.630a4ab14136bb0fd608bac65d43a423. 2023-09-29T06:20:52,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 540f2380dc3488a31274410004f1161b, disabling compactions & flushes 2023-09-29T06:20:52,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:52,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:52,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. after waiting 0 ms 2023-09-29T06:20:52,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:52,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 540f2380dc3488a31274410004f1161b 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:52,364 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/.tmp/testFamily/171ba0b3039844bea6bc37d6cbe36eba is 61, key is row-0/testFamily:testQualifier/1695968422537/Put/seqid=0 2023-09-29T06:20:52,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/.tmp/testFamily/171ba0b3039844bea6bc37d6cbe36eba 2023-09-29T06:20:52,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/.tmp/testFamily/171ba0b3039844bea6bc37d6cbe36eba as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/testFamily/171ba0b3039844bea6bc37d6cbe36eba 2023-09-29T06:20:52,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/testFamily/171ba0b3039844bea6bc37d6cbe36eba, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:52,390 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 540f2380dc3488a31274410004f1161b in 42ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:52,394 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 540f2380dc3488a31274410004f1161b, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:52,395 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/540f2380dc3488a31274410004f1161b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:52,396 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:52,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 540f2380dc3488a31274410004f1161b: 2023-09-29T06:20:52,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1695968419520.540f2380dc3488a31274410004f1161b. 2023-09-29T06:20:52,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a968f54380ef9c5e30892a9639d27324, disabling compactions & flushes 2023-09-29T06:20:52,396 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:52,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:52,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. after waiting 0 ms 2023-09-29T06:20:52,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:52,396 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing a968f54380ef9c5e30892a9639d27324 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:52,409 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/.tmp/testFamily/56e5093181894c75ab35b43a051dc425 is 61, key is row-0/testFamily:testQualifier/1695968446674/Put/seqid=0 2023-09-29T06:20:52,594 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:52,794 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 23 regions to close 2023-09-29T06:20:52,795 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, a968f54380ef9c5e30892a9639d27324, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:52,817 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/.tmp/testFamily/56e5093181894c75ab35b43a051dc425 2023-09-29T06:20:52,824 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/.tmp/testFamily/56e5093181894c75ab35b43a051dc425 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/testFamily/56e5093181894c75ab35b43a051dc425 2023-09-29T06:20:52,829 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/testFamily/56e5093181894c75ab35b43a051dc425, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:52,830 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a968f54380ef9c5e30892a9639d27324 in 434ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:52,835 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/a968f54380ef9c5e30892a9639d27324/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:52,836 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:52,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a968f54380ef9c5e30892a9639d27324: 2023-09-29T06:20:52,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1695968443655.a968f54380ef9c5e30892a9639d27324. 2023-09-29T06:20:52,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing cb052ffb687b3cbe7ab95c4d907309a7, disabling compactions & flushes 2023-09-29T06:20:52,836 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:52,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:52,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. after waiting 0 ms 2023-09-29T06:20:52,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:52,836 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing cb052ffb687b3cbe7ab95c4d907309a7 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:52,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/.tmp/testFamily/5463e102dedc400bae4859c3b10084e9 is 61, key is row-0/testFamily:testQualifier/1695968371975/Put/seqid=0 2023-09-29T06:20:52,995 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:53,195 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, cb052ffb687b3cbe7ab95c4d907309a7, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:53,261 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/.tmp/testFamily/5463e102dedc400bae4859c3b10084e9 2023-09-29T06:20:53,270 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/.tmp/testFamily/5463e102dedc400bae4859c3b10084e9 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/testFamily/5463e102dedc400bae4859c3b10084e9 2023-09-29T06:20:53,276 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/testFamily/5463e102dedc400bae4859c3b10084e9, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:53,277 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cb052ffb687b3cbe7ab95c4d907309a7 in 441ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:53,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_1_-true/cb052ffb687b3cbe7ab95c4d907309a7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:53,283 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:53,284 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for cb052ffb687b3cbe7ab95c4d907309a7: 2023-09-29T06:20:53,284 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1695968368957.cb052ffb687b3cbe7ab95c4d907309a7. 2023-09-29T06:20:53,284 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 381724feeefef7418ce9cb890fc7b15f, disabling compactions & flushes 2023-09-29T06:20:53,284 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:20:53,284 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:20:53,284 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. after waiting 0 ms 2023-09-29T06:20:53,284 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:20:53,284 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 381724feeefef7418ce9cb890fc7b15f 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:53,296 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/.tmp/testFamily/ea6238f035e94ea8b7ecf1cd5323bb3d is 61, key is row-0/testFamily:testQualifier/1695968291122/Put/seqid=0 2023-09-29T06:20:53,395 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:53,596 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 381724feeefef7418ce9cb890fc7b15f, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:53,703 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/.tmp/testFamily/ea6238f035e94ea8b7ecf1cd5323bb3d 2023-09-29T06:20:53,709 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/.tmp/testFamily/ea6238f035e94ea8b7ecf1cd5323bb3d as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/testFamily/ea6238f035e94ea8b7ecf1cd5323bb3d 2023-09-29T06:20:53,715 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/testFamily/ea6238f035e94ea8b7ecf1cd5323bb3d, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:53,716 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 381724feeefef7418ce9cb890fc7b15f in 432ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:53,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_0_-false/381724feeefef7418ce9cb890fc7b15f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:53,722 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:20:53,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 381724feeefef7418ce9cb890fc7b15f: 2023-09-29T06:20:53,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1695968288100.381724feeefef7418ce9cb890fc7b15f. 2023-09-29T06:20:53,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c5d82055d13f28d8394959085f7e9b9c, disabling compactions & flushes 2023-09-29T06:20:53,723 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:20:53,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:20:53,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. after waiting 0 ms 2023-09-29T06:20:53,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:20:53,723 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing c5d82055d13f28d8394959085f7e9b9c 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:53,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/.tmp/testFamily/79940e9f3c80443ea3421b57f9e2be7d is 61, key is row-0/testFamily:testQualifier/1695968309430/Put/seqid=0 2023-09-29T06:20:53,796 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 20 regions to close 2023-09-29T06:20:53,796 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:53,996 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c5d82055d13f28d8394959085f7e9b9c, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:54,143 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/.tmp/testFamily/79940e9f3c80443ea3421b57f9e2be7d 2023-09-29T06:20:54,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/.tmp/testFamily/79940e9f3c80443ea3421b57f9e2be7d as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/testFamily/79940e9f3c80443ea3421b57f9e2be7d 2023-09-29T06:20:54,156 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/testFamily/79940e9f3c80443ea3421b57f9e2be7d, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:54,156 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c5d82055d13f28d8394959085f7e9b9c in 433ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:54,161 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/c5d82055d13f28d8394959085f7e9b9c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:54,162 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:20:54,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c5d82055d13f28d8394959085f7e9b9c: 2023-09-29T06:20:54,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1695968308411.c5d82055d13f28d8394959085f7e9b9c. 2023-09-29T06:20:54,163 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 144c806451470faf2697726823a29c67, disabling compactions & flushes 2023-09-29T06:20:54,163 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:20:54,163 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:20:54,163 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. after waiting 0 ms 2023-09-29T06:20:54,163 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:20:54,163 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 144c806451470faf2697726823a29c67 1/1 column families, dataSize=78 B heapSize=488 B 2023-09-29T06:20:54,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/.tmp/info/7faeaaecacdb40f7bfb9338010a16969 is 45, key is default/info:d/1695968252272/Put/seqid=0 2023-09-29T06:20:54,197 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:54,397 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:54,583 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/.tmp/info/7faeaaecacdb40f7bfb9338010a16969 2023-09-29T06:20:54,590 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/.tmp/info/7faeaaecacdb40f7bfb9338010a16969 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/info/7faeaaecacdb40f7bfb9338010a16969 2023-09-29T06:20:54,595 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/info/7faeaaecacdb40f7bfb9338010a16969, entries=2, sequenceid=6, filesize=4.9 K 2023-09-29T06:20:54,596 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 144c806451470faf2697726823a29c67 in 433ms, sequenceid=6, compaction requested=false 2023-09-29T06:20:54,597 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 144c806451470faf2697726823a29c67, 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:54,602 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/namespace/144c806451470faf2697726823a29c67/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-09-29T06:20:54,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:20:54,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 144c806451470faf2697726823a29c67: 2023-09-29T06:20:54,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1695968251307.144c806451470faf2697726823a29c67. 2023-09-29T06:20:54,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3f1ecb988b69066f325faf08abfe7b96, disabling compactions & flushes 2023-09-29T06:20:54,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:20:54,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:20:54,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. after waiting 0 ms 2023-09-29T06:20:54,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:20:54,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3f1ecb988b69066f325faf08abfe7b96 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:54,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/.tmp/testFamily/54b0ea9c9b874bf4a5f4de8cd6400557 is 61, key is row-0/testFamily:testQualifier/1695968386193/Put/seqid=0 2023-09-29T06:20:54,797 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 18 regions to close 2023-09-29T06:20:54,798 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:54,998 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 3f1ecb988b69066f325faf08abfe7b96, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:55,021 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/.tmp/testFamily/54b0ea9c9b874bf4a5f4de8cd6400557 2023-09-29T06:20:55,029 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/.tmp/testFamily/54b0ea9c9b874bf4a5f4de8cd6400557 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/testFamily/54b0ea9c9b874bf4a5f4de8cd6400557 2023-09-29T06:20:55,036 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/testFamily/54b0ea9c9b874bf4a5f4de8cd6400557, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:55,037 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3f1ecb988b69066f325faf08abfe7b96 in 434ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:55,042 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNextAsync_1_-true/3f1ecb988b69066f325faf08abfe7b96/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:55,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:20:55,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3f1ecb988b69066f325faf08abfe7b96: 2023-09-29T06:20:55,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1695968383176.3f1ecb988b69066f325faf08abfe7b96. 2023-09-29T06:20:55,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2e59844a5245f32887e00ecd6717b11e, disabling compactions & flushes 2023-09-29T06:20:55,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:55,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:55,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. after waiting 0 ms 2023-09-29T06:20:55,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:55,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2e59844a5245f32887e00ecd6717b11e 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:55,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/.tmp/testFamily/a2cf4889ca49417882ffd3406b6fb6a5 is 61, key is row-0/testFamily:testQualifier/1695968434603/Put/seqid=0 2023-09-29T06:20:55,198 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:55,398 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 2e59844a5245f32887e00ecd6717b11e, 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:55,462 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/.tmp/testFamily/a2cf4889ca49417882ffd3406b6fb6a5 2023-09-29T06:20:55,469 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/.tmp/testFamily/a2cf4889ca49417882ffd3406b6fb6a5 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/testFamily/a2cf4889ca49417882ffd3406b6fb6a5 2023-09-29T06:20:55,474 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/testFamily/a2cf4889ca49417882ffd3406b6fb6a5, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:55,475 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2e59844a5245f32887e00ecd6717b11e in 432ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:55,480 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_1_-true/2e59844a5245f32887e00ecd6717b11e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:55,481 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:55,481 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2e59844a5245f32887e00ecd6717b11e: 2023-09-29T06:20:55,481 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1695968431585.2e59844a5245f32887e00ecd6717b11e. 2023-09-29T06:20:55,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d2ca20f2151278b7b07397ded57f313b, disabling compactions & flushes 2023-09-29T06:20:55,482 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:20:55,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:20:55,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. after waiting 0 ms 2023-09-29T06:20:55,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:20:55,482 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing d2ca20f2151278b7b07397ded57f313b 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:55,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/.tmp/testFamily/b2fe090514bc485884454a1be93c4267 is 61, key is row-0/testFamily:testQualifier/1695968253517/Put/seqid=0 2023-09-29T06:20:55,599 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:55,799 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 16 regions to close 2023-09-29T06:20:55,799 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, d2ca20f2151278b7b07397ded57f313b, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:55,902 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/.tmp/testFamily/b2fe090514bc485884454a1be93c4267 2023-09-29T06:20:55,909 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/.tmp/testFamily/b2fe090514bc485884454a1be93c4267 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/testFamily/b2fe090514bc485884454a1be93c4267 2023-09-29T06:20:55,915 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/testFamily/b2fe090514bc485884454a1be93c4267, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:55,916 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d2ca20f2151278b7b07397ded57f313b in 434ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:55,920 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/d2ca20f2151278b7b07397ded57f313b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:55,921 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:20:55,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d2ca20f2151278b7b07397ded57f313b: 2023-09-29T06:20:55,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1695968252464.d2ca20f2151278b7b07397ded57f313b. 2023-09-29T06:20:55,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ee80440612009a663af11a4db3cdc824, disabling compactions & flushes 2023-09-29T06:20:55,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:55,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:55,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. after waiting 0 ms 2023-09-29T06:20:55,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:55,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing ee80440612009a663af11a4db3cdc824 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:55,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/.tmp/testFamily/6beb0c04a57c4f34b93b884f7cc6f363 is 61, key is row-0/testFamily:testQualifier/1695968347830/Put/seqid=0 2023-09-29T06:20:55,999 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:56,064 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:20:56,200 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ee80440612009a663af11a4db3cdc824, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:56,341 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/.tmp/testFamily/6beb0c04a57c4f34b93b884f7cc6f363 2023-09-29T06:20:56,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/.tmp/testFamily/6beb0c04a57c4f34b93b884f7cc6f363 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/testFamily/6beb0c04a57c4f34b93b884f7cc6f363 2023-09-29T06:20:56,354 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/testFamily/6beb0c04a57c4f34b93b884f7cc6f363, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:56,355 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ee80440612009a663af11a4db3cdc824 in 433ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:56,359 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/ee80440612009a663af11a4db3cdc824/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:56,360 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:56,360 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ee80440612009a663af11a4db3cdc824: 2023-09-29T06:20:56,360 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1695968344811.ee80440612009a663af11a4db3cdc824. 2023-09-29T06:20:56,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 900e65ae3ca33070283ae33e92a0d0a7, disabling compactions & flushes 2023-09-29T06:20:56,361 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:56,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:56,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. after waiting 0 ms 2023-09-29T06:20:56,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:56,361 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 900e65ae3ca33070283ae33e92a0d0a7 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:56,378 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/.tmp/testFamily/4b36da6d266947eda333dae836f78903 is 61, key is row-0/testFamily:testQualifier/1695968380123/Put/seqid=0 2023-09-29T06:20:56,400 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:56,600 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:56,785 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/.tmp/testFamily/4b36da6d266947eda333dae836f78903 2023-09-29T06:20:56,792 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/.tmp/testFamily/4b36da6d266947eda333dae836f78903 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/testFamily/4b36da6d266947eda333dae836f78903 2023-09-29T06:20:56,797 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/testFamily/4b36da6d266947eda333dae836f78903, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:56,798 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 900e65ae3ca33070283ae33e92a0d0a7 in 437ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:56,800 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 14 regions to close 2023-09-29T06:20:56,800 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 900e65ae3ca33070283ae33e92a0d0a7, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:56,803 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/900e65ae3ca33070283ae33e92a0d0a7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:56,803 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:56,803 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 900e65ae3ca33070283ae33e92a0d0a7: 2023-09-29T06:20:56,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1695968377107.900e65ae3ca33070283ae33e92a0d0a7. 2023-09-29T06:20:56,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ffd661ca8f6efec9237d72fe94e42cb3, disabling compactions & flushes 2023-09-29T06:20:56,804 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:20:56,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:20:56,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. after waiting 0 ms 2023-09-29T06:20:56,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:20:56,804 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing ffd661ca8f6efec9237d72fe94e42cb3 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:56,816 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/.tmp/testFamily/023ba861a53c42759c7a1ba01111dc1c is 61, key is row-0/testFamily:testQualifier/1695968375050/Put/seqid=0 2023-09-29T06:20:57,001 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:57,201 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141, ffd661ca8f6efec9237d72fe94e42cb3 2023-09-29T06:20:57,222 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/.tmp/testFamily/023ba861a53c42759c7a1ba01111dc1c 2023-09-29T06:20:57,228 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/.tmp/testFamily/023ba861a53c42759c7a1ba01111dc1c as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/testFamily/023ba861a53c42759c7a1ba01111dc1c 2023-09-29T06:20:57,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/testFamily/023ba861a53c42759c7a1ba01111dc1c, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:57,234 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ffd661ca8f6efec9237d72fe94e42cb3 in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:57,239 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ffd661ca8f6efec9237d72fe94e42cb3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:57,240 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:20:57,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ffd661ca8f6efec9237d72fe94e42cb3: 2023-09-29T06:20:57,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1695968372033.ffd661ca8f6efec9237d72fe94e42cb3. 2023-09-29T06:20:57,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ca05f8d8c61d1239597333e521f0b141, disabling compactions & flushes 2023-09-29T06:20:57,240 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:57,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:57,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. after waiting 0 ms 2023-09-29T06:20:57,241 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:57,241 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing ca05f8d8c61d1239597333e521f0b141 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:57,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/.tmp/testFamily/3575f20ed9ff4cccb0842161a57fe62f is 61, key is row-0/testFamily:testQualifier/1695968406386/Put/seqid=0 2023-09-29T06:20:57,401 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:57,601 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28, ca05f8d8c61d1239597333e521f0b141 2023-09-29T06:20:57,659 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/.tmp/testFamily/3575f20ed9ff4cccb0842161a57fe62f 2023-09-29T06:20:57,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/.tmp/testFamily/3575f20ed9ff4cccb0842161a57fe62f as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/testFamily/3575f20ed9ff4cccb0842161a57fe62f 2023-09-29T06:20:57,671 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/testFamily/3575f20ed9ff4cccb0842161a57fe62f, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:57,672 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ca05f8d8c61d1239597333e521f0b141 in 431ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:57,678 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnNext_1_-true/ca05f8d8c61d1239597333e521f0b141/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:57,679 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:57,679 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ca05f8d8c61d1239597333e521f0b141: 2023-09-29T06:20:57,679 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1695968403368.ca05f8d8c61d1239597333e521f0b141. 2023-09-29T06:20:57,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 6c193cd68953fa128ebdeb920ceacb28, disabling compactions & flushes 2023-09-29T06:20:57,680 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:20:57,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:20:57,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. after waiting 0 ms 2023-09-29T06:20:57,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:20:57,680 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 6c193cd68953fa128ebdeb920ceacb28 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:57,692 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/.tmp/testFamily/45a78fae4e7c43a09e47adaffadc4e4c is 61, key is row-0/testFamily:testQualifier/1695968359898/Put/seqid=0 2023-09-29T06:20:57,802 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 11 regions to close 2023-09-29T06:20:57,802 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:58,002 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 6c193cd68953fa128ebdeb920ceacb28, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:58,099 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/.tmp/testFamily/45a78fae4e7c43a09e47adaffadc4e4c 2023-09-29T06:20:58,105 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/.tmp/testFamily/45a78fae4e7c43a09e47adaffadc4e4c as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/testFamily/45a78fae4e7c43a09e47adaffadc4e4c 2023-09-29T06:20:58,111 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/testFamily/45a78fae4e7c43a09e47adaffadc4e4c, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:58,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c193cd68953fa128ebdeb920ceacb28 in 432ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:58,117 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/6c193cd68953fa128ebdeb920ceacb28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:58,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:20:58,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 6c193cd68953fa128ebdeb920ceacb28: 2023-09-29T06:20:58,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1695968356879.6c193cd68953fa128ebdeb920ceacb28. 2023-09-29T06:20:58,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9a7d56d0a85a7d831c743aa9169ba277, disabling compactions & flushes 2023-09-29T06:20:58,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:20:58,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:20:58,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. after waiting 0 ms 2023-09-29T06:20:58,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:20:58,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9a7d56d0a85a7d831c743aa9169ba277 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:58,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/.tmp/testFamily/3cf2ff0fbb664de3840e45c2b9149834 is 61, key is row-0/testFamily:testQualifier/1695968319515/Put/seqid=0 2023-09-29T06:20:58,202 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:58,403 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9a7d56d0a85a7d831c743aa9169ba277, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:58,537 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/.tmp/testFamily/3cf2ff0fbb664de3840e45c2b9149834 2023-09-29T06:20:58,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/.tmp/testFamily/3cf2ff0fbb664de3840e45c2b9149834 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/testFamily/3cf2ff0fbb664de3840e45c2b9149834 2023-09-29T06:20:58,548 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/testFamily/3cf2ff0fbb664de3840e45c2b9149834, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:58,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9a7d56d0a85a7d831c743aa9169ba277 in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:58,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/9a7d56d0a85a7d831c743aa9169ba277/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:58,554 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:20:58,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9a7d56d0a85a7d831c743aa9169ba277: 2023-09-29T06:20:58,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1695968316495.9a7d56d0a85a7d831c743aa9169ba277. 2023-09-29T06:20:58,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3b2e2b8af1b0a36351d7765af650b765, disabling compactions & flushes 2023-09-29T06:20:58,554 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:20:58,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:20:58,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. after waiting 0 ms 2023-09-29T06:20:58,555 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:20:58,555 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3b2e2b8af1b0a36351d7765af650b765 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:58,566 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/.tmp/testFamily/0e40da57254f47a291558d5a82409bcd is 61, key is row-0/testFamily:testQualifier/1695968254668/Put/seqid=0 2023-09-29T06:20:58,603 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2023-09-29T06:20:58,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2023-09-29T06:20:58,803 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 9 regions to close 2023-09-29T06:20:58,803 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1583): Online Regions={3b2e2b8af1b0a36351d7765af650b765=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765., a5a35f3ea933f9ee2575761d0ca57b8b=testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b., b60b1ecdd7b636a2e033c7b1d93d483f=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., 58d56be7906234b32fb9db25b080a032=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032., c6e7c51cc3296370de4e4c0324ab2c28=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., 66db2bdb534f53f2003060ce8de1b2ac=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac., b7d72ea3cc8715208649befa0c48f46e=hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e., 320342231d3957a0d6191f34e62ec4a2=hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2., 9d55fd58d40728f94dca337977664f2c=testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.} 2023-09-29T06:20:58,803 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 3b2e2b8af1b0a36351d7765af650b765, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:58,972 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/.tmp/testFamily/0e40da57254f47a291558d5a82409bcd 2023-09-29T06:20:58,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/.tmp/testFamily/0e40da57254f47a291558d5a82409bcd as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/testFamily/0e40da57254f47a291558d5a82409bcd 2023-09-29T06:20:58,984 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/testFamily/0e40da57254f47a291558d5a82409bcd, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:58,985 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3b2e2b8af1b0a36351d7765af650b765 in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:58,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/3b2e2b8af1b0a36351d7765af650b765/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:58,989 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:20:58,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3b2e2b8af1b0a36351d7765af650b765: 2023-09-29T06:20:58,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1695968253646.3b2e2b8af1b0a36351d7765af650b765. 2023-09-29T06:20:58,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a5a35f3ea933f9ee2575761d0ca57b8b, disabling compactions & flushes 2023-09-29T06:20:58,990 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:20:58,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:20:58,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. after waiting 0 ms 2023-09-29T06:20:58,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:20:58,990 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing a5a35f3ea933f9ee2575761d0ca57b8b 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:59,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/.tmp/testFamily/758d766c2baa4342ab85e460e46ecbbd is 61, key is row-0/testFamily:testQualifier/1695968278878/Put/seqid=0 2023-09-29T06:20:59,003 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:59,204 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:59,404 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, a5a35f3ea933f9ee2575761d0ca57b8b, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:59,407 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/.tmp/testFamily/758d766c2baa4342ab85e460e46ecbbd 2023-09-29T06:20:59,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/.tmp/testFamily/758d766c2baa4342ab85e460e46ecbbd as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/testFamily/758d766c2baa4342ab85e460e46ecbbd 2023-09-29T06:20:59,419 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/testFamily/758d766c2baa4342ab85e460e46ecbbd, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:59,420 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a5a35f3ea933f9ee2575761d0ca57b8b in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:59,424 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testRetryOutOfOrderScannerNextException_0_-false/a5a35f3ea933f9ee2575761d0ca57b8b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:59,425 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:20:59,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a5a35f3ea933f9ee2575761d0ca57b8b: 2023-09-29T06:20:59,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1695968275856.a5a35f3ea933f9ee2575761d0ca57b8b. 2023-09-29T06:20:59,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b60b1ecdd7b636a2e033c7b1d93d483f, disabling compactions & flushes 2023-09-29T06:20:59,425 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:20:59,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:20:59,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. after waiting 0 ms 2023-09-29T06:20:59,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:20:59,425 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b60b1ecdd7b636a2e033c7b1d93d483f 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:59,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/.tmp/testFamily/1149f541c7d240fb91fee67c3abb0520 is 61, key is row-0/testFamily:testQualifier/1695968285048/Put/seqid=0 2023-09-29T06:20:59,604 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:59,804 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 7 regions to close 2023-09-29T06:20:59,804 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1583): Online Regions={b60b1ecdd7b636a2e033c7b1d93d483f=testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f., 58d56be7906234b32fb9db25b080a032=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032., c6e7c51cc3296370de4e4c0324ab2c28=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28., 66db2bdb534f53f2003060ce8de1b2ac=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac., b7d72ea3cc8715208649befa0c48f46e=hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e., 320342231d3957a0d6191f34e62ec4a2=hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2., 9d55fd58d40728f94dca337977664f2c=testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.} 2023-09-29T06:20:59,805 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b60b1ecdd7b636a2e033c7b1d93d483f, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:20:59,843 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/.tmp/testFamily/1149f541c7d240fb91fee67c3abb0520 2023-09-29T06:20:59,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/.tmp/testFamily/1149f541c7d240fb91fee67c3abb0520 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/testFamily/1149f541c7d240fb91fee67c3abb0520 2023-09-29T06:20:59,855 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/testFamily/1149f541c7d240fb91fee67c3abb0520, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:20:59,856 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b60b1ecdd7b636a2e033c7b1d93d483f in 431ms, sequenceid=8, compaction requested=false 2023-09-29T06:20:59,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/b60b1ecdd7b636a2e033c7b1d93d483f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:20:59,861 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:20:59,861 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b60b1ecdd7b636a2e033c7b1d93d483f: 2023-09-29T06:20:59,861 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1695968284030.b60b1ecdd7b636a2e033c7b1d93d483f. 2023-09-29T06:20:59,861 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 58d56be7906234b32fb9db25b080a032, disabling compactions & flushes 2023-09-29T06:20:59,861 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:20:59,861 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:20:59,861 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. after waiting 0 ms 2023-09-29T06:20:59,861 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:20:59,861 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 58d56be7906234b32fb9db25b080a032 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:20:59,874 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/.tmp/testFamily/ad3e8d3e4c284acfbebb3ca25e71a211 is 61, key is row-0/testFamily:testQualifier/1695968266760/Put/seqid=0 2023-09-29T06:21:00,005 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:21:00,205 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 58d56be7906234b32fb9db25b080a032, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:21:00,280 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/.tmp/testFamily/ad3e8d3e4c284acfbebb3ca25e71a211 2023-09-29T06:21:00,287 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/.tmp/testFamily/ad3e8d3e4c284acfbebb3ca25e71a211 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/testFamily/ad3e8d3e4c284acfbebb3ca25e71a211 2023-09-29T06:21:00,292 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/testFamily/ad3e8d3e4c284acfbebb3ca25e71a211, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:21:00,293 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 58d56be7906234b32fb9db25b080a032 in 432ms, sequenceid=8, compaction requested=false 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/58d56be7906234b32fb9db25b080a032/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:21:00,298 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 58d56be7906234b32fb9db25b080a032: 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1695968263737.58d56be7906234b32fb9db25b080a032. 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c6e7c51cc3296370de4e4c0324ab2c28, disabling compactions & flushes 2023-09-29T06:21:00,298 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. after waiting 0 ms 2023-09-29T06:21:00,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:21:00,299 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing c6e7c51cc3296370de4e4c0324ab2c28 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:21:00,310 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/.tmp/testFamily/ef2a360cbdc940b1a00fa08e025da82d is 61, key is row-0/testFamily:testQualifier/1695968338680/Put/seqid=0 2023-09-29T06:21:00,405 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:21:00,546 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2023-09-29T06:21:00,606 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e, c6e7c51cc3296370de4e4c0324ab2c28 2023-09-29T06:21:00,716 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/.tmp/testFamily/ef2a360cbdc940b1a00fa08e025da82d 2023-09-29T06:21:00,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/.tmp/testFamily/ef2a360cbdc940b1a00fa08e025da82d as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/testFamily/ef2a360cbdc940b1a00fa08e025da82d 2023-09-29T06:21:00,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/testFamily/ef2a360cbdc940b1a00fa08e025da82d, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:21:00,729 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c6e7c51cc3296370de4e4c0324ab2c28 in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:21:00,733 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c6e7c51cc3296370de4e4c0324ab2c28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:21:00,734 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:21:00,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c6e7c51cc3296370de4e4c0324ab2c28: 2023-09-29T06:21:00,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1695968335661.c6e7c51cc3296370de4e4c0324ab2c28. 2023-09-29T06:21:00,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 66db2bdb534f53f2003060ce8de1b2ac, disabling compactions & flushes 2023-09-29T06:21:00,734 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:21:00,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:21:00,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. after waiting 0 ms 2023-09-29T06:21:00,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:21:00,734 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 66db2bdb534f53f2003060ce8de1b2ac 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:21:00,746 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/.tmp/testFamily/0e78d4b23ce749b69c33ba87133d79fd is 61, key is row-0/testFamily:testQualifier/1695968412458/Put/seqid=0 2023-09-29T06:21:00,806 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-09-29T06:21:00,806 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1583): Online Regions={66db2bdb534f53f2003060ce8de1b2ac=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac., b7d72ea3cc8715208649befa0c48f46e=hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e., 320342231d3957a0d6191f34e62ec4a2=hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2., 9d55fd58d40728f94dca337977664f2c=testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.} 2023-09-29T06:21:00,806 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:21:00,983 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:21:01,006 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 66db2bdb534f53f2003060ce8de1b2ac, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:21:01,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/.tmp/testFamily/0e78d4b23ce749b69c33ba87133d79fd 2023-09-29T06:21:01,159 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/.tmp/testFamily/0e78d4b23ce749b69c33ba87133d79fd as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/testFamily/0e78d4b23ce749b69c33ba87133d79fd 2023-09-29T06:21:01,164 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/testFamily/0e78d4b23ce749b69c33ba87133d79fd, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:21:01,165 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66db2bdb534f53f2003060ce8de1b2ac in 431ms, sequenceid=8, compaction requested=false 2023-09-29T06:21:01,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66db2bdb534f53f2003060ce8de1b2ac/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:21:01,170 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:21:01,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 66db2bdb534f53f2003060ce8de1b2ac: 2023-09-29T06:21:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1695968409439.66db2bdb534f53f2003060ce8de1b2ac. 2023-09-29T06:21:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b7d72ea3cc8715208649befa0c48f46e, disabling compactions & flushes 2023-09-29T06:21:01,171 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:21:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:21:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. after waiting 0 ms 2023-09-29T06:21:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:21:01,171 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b7d72ea3cc8715208649befa0c48f46e 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:21:01,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/.tmp/testFamily/bd990e06cdf9424aa40f9174c5678b53 is 61, key is row-0/testFamily:testQualifier/1695968329589/Put/seqid=0 2023-09-29T06:21:01,206 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:21:01,407 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 9d55fd58d40728f94dca337977664f2c, b7d72ea3cc8715208649befa0c48f46e 2023-09-29T06:21:01,589 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/.tmp/testFamily/bd990e06cdf9424aa40f9174c5678b53 2023-09-29T06:21:01,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/.tmp/testFamily/bd990e06cdf9424aa40f9174c5678b53 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/testFamily/bd990e06cdf9424aa40f9174c5678b53 2023-09-29T06:21:01,600 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/testFamily/bd990e06cdf9424aa40f9174c5678b53, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:21:01,601 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b7d72ea3cc8715208649befa0c48f46e in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:21:01,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnNext_0_-false/b7d72ea3cc8715208649befa0c48f46e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:21:01,607 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:21:01,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b7d72ea3cc8715208649befa0c48f46e: 2023-09-29T06:21:01,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1695968328571.b7d72ea3cc8715208649befa0c48f46e. 2023-09-29T06:21:01,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 320342231d3957a0d6191f34e62ec4a2, disabling compactions & flushes 2023-09-29T06:21:01,607 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:21:01,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:21:01,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. after waiting 0 ms 2023-09-29T06:21:01,607 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:21:01,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:21:01,607 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 320342231d3957a0d6191f34e62ec4a2 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:21:01,619 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/.tmp/testFamily/c149089438c24d8191d2e2a614fb2194 is 61, key is row-0/testFamily:testQualifier/1695968295202/Put/seqid=0 2023-09-29T06:21:01,807 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-09-29T06:21:01,807 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1583): Online Regions={320342231d3957a0d6191f34e62ec4a2=hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2., 9d55fd58d40728f94dca337977664f2c=testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c.} 2023-09-29T06:21:01,807 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:21:02,007 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 320342231d3957a0d6191f34e62ec4a2, 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:21:02,025 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/.tmp/testFamily/c149089438c24d8191d2e2a614fb2194 2023-09-29T06:21:02,031 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/.tmp/testFamily/c149089438c24d8191d2e2a614fb2194 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/testFamily/c149089438c24d8191d2e2a614fb2194 2023-09-29T06:21:02,036 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/testFamily/c149089438c24d8191d2e2a614fb2194, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:21:02,037 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 320342231d3957a0d6191f34e62ec4a2 in 430ms, sequenceid=8, compaction requested=false 2023-09-29T06:21:02,042 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/hbase/testMetaScanTimeoutOnClose_0_-false/320342231d3957a0d6191f34e62ec4a2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:21:02,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:21:02,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 320342231d3957a0d6191f34e62ec4a2: 2023-09-29T06:21:02,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1695968294182.320342231d3957a0d6191f34e62ec4a2. 2023-09-29T06:21:02,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9d55fd58d40728f94dca337977664f2c, disabling compactions & flushes 2023-09-29T06:21:02,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:21:02,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:21:02,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. after waiting 0 ms 2023-09-29T06:21:02,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:21:02,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9d55fd58d40728f94dca337977664f2c 1/1 column families, dataSize=228 B heapSize=800 B 2023-09-29T06:21:02,055 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/.tmp/testFamily/30e00f490c994991bd66157d71ae7b88 is 61, key is row-0/testFamily:testQualifier/1695968302771/Put/seqid=0 2023-09-29T06:21:02,208 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:21:02,408 DEBUG [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1609): Waiting on 9d55fd58d40728f94dca337977664f2c 2023-09-29T06:21:02,462 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/.tmp/testFamily/30e00f490c994991bd66157d71ae7b88 2023-09-29T06:21:02,468 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/.tmp/testFamily/30e00f490c994991bd66157d71ae7b88 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/testFamily/30e00f490c994991bd66157d71ae7b88 2023-09-29T06:21:02,473 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/testFamily/30e00f490c994991bd66157d71ae7b88, entries=4, sequenceid=8, filesize=4.9 K 2023-09-29T06:21:02,474 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9d55fd58d40728f94dca337977664f2c in 431ms, sequenceid=8, compaction requested=false 2023-09-29T06:21:02,479 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/data/default/testNormalScanTimeoutOnClose_0_-false/9d55fd58d40728f94dca337977664f2c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-09-29T06:21:02,480 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:21:02,480 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9d55fd58d40728f94dca337977664f2c: 2023-09-29T06:21:02,480 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1695968301751.9d55fd58d40728f94dca337977664f2c. 2023-09-29T06:21:02,608 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43651,1695968188964; all regions closed. 2023-09-29T06:21:02,619 DEBUG [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/oldWALs 2023-09-29T06:21:02,619 INFO [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C43651%2C1695968188964.meta:.meta(num 1695968251072) 2023-09-29T06:21:02,624 DEBUG [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/oldWALs 2023-09-29T06:21:02,624 INFO [RS:0;jenkins-hbase3:43651 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C43651%2C1695968188964:(num 1695968250595) 2023-09-29T06:21:02,624 DEBUG [RS:0;jenkins-hbase3:43651 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:21:02,624 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.LeaseManager(133): Closed leases 2023-09-29T06:21:02,625 INFO [RS:0;jenkins-hbase3:43651 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-09-29T06:21:02,625 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-29T06:21:02,626 INFO [RS:0;jenkins-hbase3:43651 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:43651 2023-09-29T06:21:02,631 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/jenkins-hbase3.apache.org,43651,1695968188964 2023-09-29T06:21:02,631 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-09-29T06:21:02,634 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,43651,1695968188964] 2023-09-29T06:21:02,634 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,43651,1695968188964; numProcessing=1 2023-09-29T06:21:02,635 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/draining/jenkins-hbase3.apache.org,43651,1695968188964 already deleted, retry=false 2023-09-29T06:21:02,635 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,43651,1695968188964 expired; onlineServers=0 2023-09-29T06:21:02,635 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,38609,1695968187790' ***** 2023-09-29T06:21:02,635 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-29T06:21:02,636 DEBUG [M:0;jenkins-hbase3:38609 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758488c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-29T06:21:02,636 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,38609,1695968187790 2023-09-29T06:21:02,636 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,38609,1695968187790; all regions closed. 2023-09-29T06:21:02,636 DEBUG [M:0;jenkins-hbase3:38609 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:21:02,636 DEBUG [M:0;jenkins-hbase3:38609 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-29T06:21:02,636 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-29T06:21:02,636 DEBUG [M:0;jenkins-hbase3:38609 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-29T06:21:02,636 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1695968190418 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1695968190418,5,FailOnTimeoutGroup] 2023-09-29T06:21:02,636 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1695968190419 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1695968190419,5,FailOnTimeoutGroup] 2023-09-29T06:21:02,637 INFO [M:0;jenkins-hbase3:38609 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-29T06:21:02,637 DEBUG [M:0;jenkins-hbase3:38609 {}] master.HMaster(1688): Stopping service threads 2023-09-29T06:21:02,637 INFO [M:0;jenkins-hbase3:38609 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-29T06:21:02,637 ERROR [M:0;jenkins-hbase3:38609 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-09-29T06:21:02,638 INFO [M:0;jenkins-hbase3:38609 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-29T06:21:02,638 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-29T06:21:02,638 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2023-09-29T06:21:02,638 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:21:02,638 DEBUG [M:0;jenkins-hbase3:38609 {}] zookeeper.ZKUtil(347): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2023-09-29T06:21:02,638 WARN [M:0;jenkins-hbase3:38609 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-29T06:21:02,638 INFO [M:0;jenkins-hbase3:38609 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-29T06:21:02,638 INFO [M:0;jenkins-hbase3:38609 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-29T06:21:02,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-29T06:21:02,639 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-29T06:21:02,639 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:21:02,639 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:21:02,639 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-29T06:21:02,639 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:21:02,639 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=451.83 KB heapSize=523.08 KB 2023-09-29T06:21:02,652 DEBUG [M:0;jenkins-hbase3:38609 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fccfe790d81f4665bac61d3cf49ac8c7 is 88, key is hbase:meta,,1/info:sn/1695968250838/Put/seqid=0 2023-09-29T06:21:02,699 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-29T06:21:02,734 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:21:02,734 INFO [RS:0;jenkins-hbase3:43651 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43651,1695968188964; zookeeper connection closed. 2023-09-29T06:21:02,734 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43651-0x101625e326e0001, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:21:02,734 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e39e6b3 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e39e6b3 2023-09-29T06:21:02,735 INFO [Listener at localhost/43419 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-29T06:21:03,057 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=828 (bloomFilter=true), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fccfe790d81f4665bac61d3cf49ac8c7 2023-09-29T06:21:03,084 DEBUG [M:0;jenkins-hbase3:38609 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8fa8aede3d44b27ae609c7150d96e4d is 828, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1695968253283/Put/seqid=0 2023-09-29T06:21:03,490 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451.23 KB at sequenceid=828 (bloomFilter=true), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8fa8aede3d44b27ae609c7150d96e4d 2023-09-29T06:21:03,496 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e8fa8aede3d44b27ae609c7150d96e4d 2023-09-29T06:21:03,513 DEBUG [M:0;jenkins-hbase3:38609 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9da454ec3f714f4e9fc7fa1bc993210a is 82, key is jenkins-hbase3.apache.org,43651,1695968188964/rs:state/1695968220466/Put/seqid=0 2023-09-29T06:21:03,918 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=828 (bloomFilter=true), to=hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9da454ec3f714f4e9fc7fa1bc993210a 2023-09-29T06:21:03,924 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fccfe790d81f4665bac61d3cf49ac8c7 as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fccfe790d81f4665bac61d3cf49ac8c7 2023-09-29T06:21:03,929 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fccfe790d81f4665bac61d3cf49ac8c7, entries=8, sequenceid=828, filesize=5.5 K 2023-09-29T06:21:03,930 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8fa8aede3d44b27ae609c7150d96e4d as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8fa8aede3d44b27ae609c7150d96e4d 2023-09-29T06:21:03,934 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e8fa8aede3d44b27ae609c7150d96e4d 2023-09-29T06:21:03,934 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8fa8aede3d44b27ae609c7150d96e4d, entries=92, sequenceid=828, filesize=29.5 K 2023-09-29T06:21:03,935 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9da454ec3f714f4e9fc7fa1bc993210a as hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9da454ec3f714f4e9fc7fa1bc993210a 2023-09-29T06:21:03,939 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45753/user/jenkins/test-data/3f0a3e9f-f7cb-5b8f-f757-759f5feef606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9da454ec3f714f4e9fc7fa1bc993210a, entries=1, sequenceid=828, filesize=5.1 K 2023-09-29T06:21:03,940 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(3022): Finished flush of dataSize ~451.83 KB/462669, heapSize ~522.78 KB/535328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1301ms, sequenceid=828, compaction requested=false 2023-09-29T06:21:03,942 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:21:03,943 DEBUG [M:0;jenkins-hbase3:38609 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-29T06:21:03,945 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-29T06:21:03,945 INFO [M:0;jenkins-hbase3:38609 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-29T06:21:03,945 INFO [M:0;jenkins-hbase3:38609 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:38609 2023-09-29T06:21:03,947 DEBUG [M:0;jenkins-hbase3:38609 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/rs/jenkins-hbase3.apache.org,38609,1695968187790 already deleted, retry=false 2023-09-29T06:21:04,050 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:21:04,050 INFO [M:0;jenkins-hbase3:38609 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,38609,1695968187790; zookeeper connection closed. 2023-09-29T06:21:04,050 DEBUG [Listener at localhost/43419-EventThread {}] zookeeper.ZKWatcher(604): master:38609-0x101625e326e0000, quorum=127.0.0.1:62866, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:21:04,051 WARN [Listener at localhost/43419 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-29T06:21:04,055 INFO [Listener at localhost/43419 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-29T06:21:04,158 WARN [BP-150475455-172.31.12.81-1695968184797 heartbeating to localhost/127.0.0.1:45753 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-29T06:21:04,158 WARN [BP-150475455-172.31.12.81-1695968184797 heartbeating to localhost/127.0.0.1:45753 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-150475455-172.31.12.81-1695968184797 (Datanode Uuid 2e3782e2-0094-4bd8-b140-3621a96bf897) service to localhost/127.0.0.1:45753 2023-09-29T06:21:04,159 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/cluster_77b02200-3a07-81cb-6660-8009703e16e4/dfs/data/data1/current/BP-150475455-172.31.12.81-1695968184797 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-29T06:21:04,160 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9decddf3-964e-71d1-ef1e-f567d55ea000/cluster_77b02200-3a07-81cb-6660-8009703e16e4/dfs/data/data2/current/BP-150475455-172.31.12.81-1695968184797 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-29T06:21:04,182 INFO [Listener at localhost/43419 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-29T06:21:04,298 INFO [Listener at localhost/43419 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-09-29T06:21:04,345 INFO [Listener at localhost/43419 {}] hbase.HBaseTestingUtility(1347): Minicluster is down